diff --git a/rdagent/log/timer.py b/rdagent/log/timer.py index 62716734..bcf252e5 100644 --- a/rdagent/log/timer.py +++ b/rdagent/log/timer.py @@ -75,10 +75,12 @@ class RDAgentTimer: class RDAgentTimerWrapper(SingletonBaseClass): def __init__(self) -> None: self.timer: RDAgentTimer = RDAgentTimer() + self.api_fail_count: int = 0 + self.latest_api_fail_time: datetime | None = None def replace_timer(self, timer: RDAgentTimer) -> None: self.timer = timer logger.info("Timer replaced successfully.") -RD_Agent_TIMER_wrapper = RDAgentTimerWrapper() +RD_Agent_TIMER_wrapper: RDAgentTimerWrapper = RDAgentTimerWrapper() diff --git a/rdagent/oai/backend/base.py b/rdagent/oai/backend/base.py index f00a568e..357f0f86 100644 --- a/rdagent/oai/backend/base.py +++ b/rdagent/oai/backend/base.py @@ -11,6 +11,7 @@ from datetime import datetime from pathlib import Path from typing import Any, Optional, cast +import pytz from pydantic import TypeAdapter from rdagent.core.utils import LLM_CACHE_SEED_GEN, SingletonBaseClass @@ -349,6 +350,8 @@ class APIBackend(ABC): content[: len(content) // 2] for content in kwargs.get("input_content_list", []) ] else: + RD_Agent_TIMER_wrapper.api_fail_count += 1 + RD_Agent_TIMER_wrapper.latest_api_fail_time = datetime.now(pytz.timezone("Asia/Shanghai")) if ( openai_imported and isinstance(e, openai.APITimeoutError) diff --git a/rdagent/utils/workflow.py b/rdagent/utils/workflow.py index dd9b243f..721ba9e0 100644 --- a/rdagent/utils/workflow.py +++ b/rdagent/utils/workflow.py @@ -138,6 +138,20 @@ class LoopBase: + current_local_datetime.year * 1e10 ) mlflow.log_metric("current_datetime", float_like_datetime) + mlflow.log_metric("api_fail_count", RD_Agent_TIMER_wrapper.api_fail_count) + lastest_api_fail_time = RD_Agent_TIMER_wrapper.latest_api_fail_time + if lastest_api_fail_time is not None: + mlflow.log_metric( + "api_fail_count", + ( + lastest_api_fail_time.second + + lastest_api_fail_time.minute * 1e2 + + lastest_api_fail_time.hour * 1e4 + + lastest_api_fail_time.day * 1e6 + + lastest_api_fail_time.month * 1e8 + + lastest_api_fail_time.year * 1e10 + ), + ) if self.timer.started: if RD_AGENT_SETTINGS.enable_mlflow: