mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-28 07:57:44 +00:00
feat: log api status to mlflow (#860)
* log api status to mlflow to frontend tracking * fix CI --------- Co-authored-by: Xu Yang <xuyang1@microsoft.com>
This commit is contained in:
@@ -75,10 +75,12 @@ class RDAgentTimer:
|
||||
class RDAgentTimerWrapper(SingletonBaseClass):
|
||||
def __init__(self) -> None:
|
||||
self.timer: RDAgentTimer = RDAgentTimer()
|
||||
self.api_fail_count: int = 0
|
||||
self.latest_api_fail_time: datetime | None = None
|
||||
|
||||
def replace_timer(self, timer: RDAgentTimer) -> None:
|
||||
self.timer = timer
|
||||
logger.info("Timer replaced successfully.")
|
||||
|
||||
|
||||
RD_Agent_TIMER_wrapper = RDAgentTimerWrapper()
|
||||
RD_Agent_TIMER_wrapper: RDAgentTimerWrapper = RDAgentTimerWrapper()
|
||||
|
||||
@@ -11,6 +11,7 @@ from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, cast
|
||||
|
||||
import pytz
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from rdagent.core.utils import LLM_CACHE_SEED_GEN, SingletonBaseClass
|
||||
@@ -349,6 +350,8 @@ class APIBackend(ABC):
|
||||
content[: len(content) // 2] for content in kwargs.get("input_content_list", [])
|
||||
]
|
||||
else:
|
||||
RD_Agent_TIMER_wrapper.api_fail_count += 1
|
||||
RD_Agent_TIMER_wrapper.latest_api_fail_time = datetime.now(pytz.timezone("Asia/Shanghai"))
|
||||
if (
|
||||
openai_imported
|
||||
and isinstance(e, openai.APITimeoutError)
|
||||
|
||||
@@ -138,6 +138,20 @@ class LoopBase:
|
||||
+ current_local_datetime.year * 1e10
|
||||
)
|
||||
mlflow.log_metric("current_datetime", float_like_datetime)
|
||||
mlflow.log_metric("api_fail_count", RD_Agent_TIMER_wrapper.api_fail_count)
|
||||
lastest_api_fail_time = RD_Agent_TIMER_wrapper.latest_api_fail_time
|
||||
if lastest_api_fail_time is not None:
|
||||
mlflow.log_metric(
|
||||
"api_fail_count",
|
||||
(
|
||||
lastest_api_fail_time.second
|
||||
+ lastest_api_fail_time.minute * 1e2
|
||||
+ lastest_api_fail_time.hour * 1e4
|
||||
+ lastest_api_fail_time.day * 1e6
|
||||
+ lastest_api_fail_time.month * 1e8
|
||||
+ lastest_api_fail_time.year * 1e10
|
||||
),
|
||||
)
|
||||
|
||||
if self.timer.started:
|
||||
if RD_AGENT_SETTINGS.enable_mlflow:
|
||||
|
||||
Reference in New Issue
Block a user