gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 23:47:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: fix chat_max_tokens calculation method to show true input_max_tokens (#1241)

Browse Source 
* fix: use real max_input_length

* lint

* Update rdagent/oai/backend/litellm.py

Co-authored-by: you-n-g <you-n-g@users.noreply.github.com>

* fix lint

* lint

* lint

---------

Co-authored-by: you-n-g <you-n-g@users.noreply.github.com>
This commit is contained in:
amstrongzyf
2025-09-12 15:09:20 +08:00
committed by GitHub
parent 4de999c4a3
commit 7f94e3a9c3
2 changed files with 14 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/oai/backend/deprec.py
+1 -1
View File
@@ -153,7 +153,7 @@ class DeprecBackend(APIBackend):
self.gcr_endpoint_do_sample = LLM_SETTINGS.gcr_endpoint_do_sample
self.gcr_endpoint_max_token = LLM_SETTINGS.gcr_endpoint_max_token
if not os.environ.get("PYTHONHTTPSVERIFY", "") and hasattr(ssl, "_create_unverified_context"):
ssl._create_default_https_context = ssl._create_unverified_context # noqa: SLF001
ssl._create_default_https_context = ssl._create_unverified_context # type: ignore[assignment]
self.chat_model_map = LLM_SETTINGS.chat_model_map
self.chat_model = LLM_SETTINGS.chat_model
self.encoder = None
rdagent/oai/backend/litellm.py
+13 -4
View File
@@ -7,7 +7,7 @@ from litellm import (
completion,
completion_cost,
embedding,
get_max_tokens,
get_model_info,
supports_function_calling,
supports_response_schema,
token_counter,
@@ -205,10 +205,19 @@ class LiteLLMAPIBackend(APIBackend):
@property
def chat_token_limit(self) -> int:
"""Suggest an input token limit, ensuring enough space in the context window for the maximum output tokens."""
try:
max_tokens = get_max_tokens(LITELLM_SETTINGS.chat_model)
if max_tokens is None:
model_info = get_model_info(LITELLM_SETTINGS.chat_model)
if model_info is None:
return super().chat_token_limit
return max_tokens
max_input = model_info.get("max_input_tokens")
max_output = model_info.get("max_output_tokens")
if max_input is None or max_output is None:
return super().chat_token_limit
max_input_tokens = max_input - max_output
return max_input_tokens
except Exception as e:
return super().chat_token_limit