diff --git a/rdagent/oai/backend/deprec.py b/rdagent/oai/backend/deprec.py index 3fcae490..adfdbc8d 100644 --- a/rdagent/oai/backend/deprec.py +++ b/rdagent/oai/backend/deprec.py @@ -153,7 +153,7 @@ class DeprecBackend(APIBackend): self.gcr_endpoint_do_sample = LLM_SETTINGS.gcr_endpoint_do_sample self.gcr_endpoint_max_token = LLM_SETTINGS.gcr_endpoint_max_token if not os.environ.get("PYTHONHTTPSVERIFY", "") and hasattr(ssl, "_create_unverified_context"): - ssl._create_default_https_context = ssl._create_unverified_context # noqa: SLF001 + ssl._create_default_https_context = ssl._create_unverified_context # type: ignore[assignment] self.chat_model_map = LLM_SETTINGS.chat_model_map self.chat_model = LLM_SETTINGS.chat_model self.encoder = None diff --git a/rdagent/oai/backend/litellm.py b/rdagent/oai/backend/litellm.py index dd55ceaf..164b4827 100644 --- a/rdagent/oai/backend/litellm.py +++ b/rdagent/oai/backend/litellm.py @@ -7,7 +7,7 @@ from litellm import ( completion, completion_cost, embedding, - get_max_tokens, + get_model_info, supports_function_calling, supports_response_schema, token_counter, @@ -205,10 +205,19 @@ class LiteLLMAPIBackend(APIBackend): @property def chat_token_limit(self) -> int: + """Suggest an input token limit, ensuring enough space in the context window for the maximum output tokens.""" try: - max_tokens = get_max_tokens(LITELLM_SETTINGS.chat_model) - if max_tokens is None: + model_info = get_model_info(LITELLM_SETTINGS.chat_model) + if model_info is None: return super().chat_token_limit - return max_tokens + + max_input = model_info.get("max_input_tokens") + max_output = model_info.get("max_output_tokens") + + if max_input is None or max_output is None: + return super().chat_token_limit + + max_input_tokens = max_input - max_output + return max_input_tokens except Exception as e: return super().chat_token_limit