From 9890bb4b00238dfc4ef6f39b43e62e03fe7ef11c Mon Sep 17 00:00:00 2001 From: amstrongzyf <201840057@smail.nju.edu.cn> Date: Fri, 13 Jun 2025 18:32:39 +0800 Subject: [PATCH] docs: update explanation for separate config use in litellm (#958) * docs: update explanation for separate config use in litellm * docs: update default backend to `rdagent.oai.backend.LiteLLMAPIBackend` * docs: update .rst format * Update installation_and_configuration.rst --- .env.example | 50 +++++++++++++++++----- README.md | 28 ++++++++++-- docs/installation_and_configuration.rst | 52 +++++++++++++++++------ rdagent/oai/llm_conf.py | 2 +- test/oai/test_embedding_and_similarity.py | 1 + 5 files changed, 106 insertions(+), 27 deletions(-) diff --git a/.env.example b/.env.example index e43e8d0b..89b5b439 100644 --- a/.env.example +++ b/.env.example @@ -7,25 +7,53 @@ For more information about configuration options, please refer to the documentat """ +# ========================================== # Global configs: -USE_AZURE=False -CHAT_USE_AZURE_TOKEN_PROVIDER=False -EMBEDDING_USE_AZURE_TOKEN_PROVIDER=False MAX_RETRY=10 RETRY_WAIT_SECONDS=20 +# ========================================== -# LLM API Setting: -OPENAI_API_KEY= -CHAT_MODEL=gpt-4-turbo -CHAT_MAX_TOKENS=3000 -CHAT_TEMPERATURE=0.7 + +# ========================================== +# Backend Configuration +# ========================================== +# BACKEND=rdagent.oai.backend.LiteLLMAPIBackend +# ========================================== + +# ========================================== +# Backend Configuration (choose one) +# ========================================== + +# 1. Set universal API key +# CHAT_MODEL="gpt-4o" +# EMBEDDING_MODEL="text-embedding-3-small" +# OPENAI_API_BASE="https://your-endpoint.com/v1" +# OPENAI_API_KEY="sk-your-api-key-here" + +# 2. Set separate API KEY +# Chat configuration +OPENAI_API_KEY="sk-chat-key" +OPENAI_API_BASE="https://xxx-litellm.com/v1" +CHAT_MODEL='gpt-4o' + +# Embedding configuration (using other service) +# Use siliconflow as example, pay attention to the litellm_proxy prefix +LITELLM_PROXY_API_KEY="sk-embedding-service-key" +LITELLM_PROXY_API_BASE="https://api.siliconflow.cn/v1" +EMBEDDING_MODEL="litellm_proxy/BAAI/bge-large-en-v1.5" +# ========================================== + +# ========================================== +# Other Configuration +# ========================================== # CHAT_AZURE_API_BASE= # CHAT_AZURE_API_VERSION= -EMBEDDING_MODEL=text-embedding-3-small # EMBEDDING_AZURE_API_BASE= # EMBEDDING_AZURE_API_VERSION= # Cache Setting (Optional): - -# Senario Configs: \ No newline at end of file +# USE_CHAT_CACHE=True +# USE_EMBEDDING_CACHE=True +# Senario Configs: +# ========================================== \ No newline at end of file diff --git a/README.md b/README.md index 1c2b24e1..ba6801bc 100644 --- a/README.md +++ b/README.md @@ -153,16 +153,38 @@ Ensure the current user can run Docker commands **without using sudo**. You can You can set your Chat Model and Embedding Model in the following ways: -- **Using LiteLLM (Recommended)**: We now support LiteLLM as a backend for integration with multiple LLM providers. You can configure as follows: +- **Using LiteLLM (Default)**: We now support LiteLLM as a backend for integration with multiple LLM providers. You can configure in two ways: + + **Option 1: Unified API base for both models** ```bash cat << EOF > .env - BACKEND=rdagent.oai.backend.LiteLLMAPIBackend # Set to any model supported by LiteLLM. CHAT_MODEL=gpt-4o EMBEDDING_MODEL=text-embedding-3-small - # Then configure the environment variables required by your chosen model in the convention of LiteLLM here. + # Configure unified API base + OPENAI_API_BASE= OPENAI_API_KEY= ``` + + **Option 2: Separate API bases for Chat and Embedding models** + ```bash + cat << EOF > .env + # Set to any model supported by LiteLLM. + # Configure separate API bases for chat and embedding + + # CHAT MODEL: + CHAT_MODEL=gpt-4o + OPENAI_API_BASE= + OPENAI_API_KEY= + + # EMBEDDING MODEL: + # TAKE siliconflow as an example, you can use other providers. + # Note: embedding requires litellm_proxy prefix + EMBEDDING_MODEL=litellm_proxy/BAAI/bge-large-en-v1.5 + LITELLM_PROXY_API_KEY= + LITELLM_PROXY_API_BASE=https://api.siliconflow.cn/v1 + ``` + Notice: If you are using reasoning models that include thought processes in their responses (such as \ tags), you need to set the following environment variable: ```bash REASONING_THINK_RM=True diff --git a/docs/installation_and_configuration.rst b/docs/installation_and_configuration.rst index 84218df8..f5b630cc 100644 --- a/docs/installation_and_configuration.rst +++ b/docs/installation_and_configuration.rst @@ -13,30 +13,58 @@ Installation **Install Docker**: RDAgent is designed for research and development, acting like a human researcher and developer. It can write and run code in various environments, primarily using Docker for code execution. This keeps the remaining dependencies simple. Users must ensure Docker is installed before attempting most scenarios. Please refer to the `official 🐳Docker page `_ for installation instructions. Ensure the current user can run Docker commands **without using sudo**. You can verify this by executing `docker run hello-world`. -LiteLLM Backend Configuration -============================= +LiteLLM Backend Configuration (Default) +======================================= -Please create a `.env` file in the root directory of the project and add environment variables. - -Here is a sample configuration for using OpenAI's gpt-4o via LiteLLM. +Option 1: Unified API base for both models +------------------------------------------ .. code-block:: Properties - BACKEND=rdagent.oai.backend.LiteLLMAPIBackend - # It can be modified to any model supported by LiteLLM. - CHAT_MODEL=gpt-4o + # Set to any model supported by LiteLLM. + CHAT_MODEL=gpt-4o EMBEDDING_MODEL=text-embedding-3-small + # Configure unified API base # The backend api_key fully follows the convention of litellm. + OPENAI_API_BASE= OPENAI_API_KEY= +Option 2: Separate API bases for Chat and Embedding models +---------------------------------------------------------- + + .. code-block:: Properties + + # Set to any model supported by LiteLLM. + + # CHAT MODEL: + CHAT_MODEL=gpt-4o + OPENAI_API_BASE= + OPENAI_API_KEY= + + # EMBEDDING MODEL: + # TAKE siliconflow as an example, you can use other providers. + # Note: embedding requires litellm_proxy prefix + EMBEDDING_MODEL=litellm_proxy/BAAI/bge-large-en-v1.5 + LITELLM_PROXY_API_KEY= + LITELLM_PROXY_API_BASE=https://api.siliconflow.cn/v1 + Necessary parameters include: -- `BACKEND`: The backend to use. The default is `rdagent.oai.backend.DeprecBackend`. To use the LiteLLM backend, set it to `rdagent.oai.backend.LiteLLMAPIBackend`. - -- `CHAT_MODEL`: The model name of the chat model. +- `CHAT_MODEL`: The model name of the chat model. - `EMBEDDING_MODEL`: The model name of the embedding model. +- `OPENAI_API_BASE`: The base URL of the API. If `EMBEDDING_MODEL` does not start with `litellm_proxy/`, this is used for both chat and embedding models; otherwise, it is used for `CHAT_MODEL` only. + +Optional parameters (required if your embedding model is provided by a different provider than `CHAT_MODEL`): + +- `LITELLM_PROXY_API_KEY`: The API key for the embedding model, required if `EMBEDDING_MODEL` starts with `litellm_proxy/`. + +- `LITELLM_PROXY_API_BASE`: The base URL for the embedding model, required if `EMBEDDING_MODEL` starts with `litellm_proxy/`. + +**Note:** If you are using an embedding model from a provider different from the chat model, remember to add the `litellm_proxy/` prefix to the `EMBEDDING_MODEL` name. + + The `CHAT_MODEL` and `EMBEDDING_MODEL` parameters will be passed into LiteLLM's completion function. Therefore, when utilizing models provided by different providers, first review the interface configuration of LiteLLM. The model names must match those allowed by LiteLLM. @@ -65,7 +93,7 @@ Configuration(deprecated) To run the application, please create a `.env` file in the root directory of the project and add environment variables according to your requirements. -The standard configuration options for the user using the OpenAI API are provided in the `.env.example` file. +If you are using this deprecated version, you should set `BACKEND` to `rdagent.oai.backend.DeprecBackend`. Here are some other configuration options that you can use: diff --git a/rdagent/oai/llm_conf.py b/rdagent/oai/llm_conf.py index b0c0ee40..aa5f101d 100644 --- a/rdagent/oai/llm_conf.py +++ b/rdagent/oai/llm_conf.py @@ -10,7 +10,7 @@ from rdagent.core.conf import ExtendedBaseSettings class LLMSettings(ExtendedBaseSettings): # backend - backend: str = "rdagent.oai.backend.DeprecBackend" + backend: str = "rdagent.oai.backend.LiteLLMAPIBackend" chat_model: str = "gpt-4-turbo" embedding_model: str = "text-embedding-3-small" diff --git a/test/oai/test_embedding_and_similarity.py b/test/oai/test_embedding_and_similarity.py index 08cc51b3..8c12b918 100644 --- a/test/oai/test_embedding_and_similarity.py +++ b/test/oai/test_embedding_and_similarity.py @@ -24,6 +24,7 @@ class TestEmbedding(unittest.TestCase): assert similarity is not None assert isinstance(similarity, float) min_similarity_threshold = 0.8 + print(f"similarity: {similarity}") assert similarity >= min_similarity_threshold