From 6e8f91296831dafa0f3dca2bebfddc2092f0766e Mon Sep 17 00:00:00 2001 From: cyncyw <47289405+taozhiwang@users.noreply.github.com> Date: Tue, 30 Jul 2024 18:06:48 +0800 Subject: [PATCH] Openai api & eval debug (#124) * Openai api & eval debug --------- Co-authored-by: Young Co-authored-by: you-n-g --- .env.example | 19 +----- README.md | 4 +- docs/installation.rst | 38 ++++++++++-- rdagent/app/quant_factor_benchmark/eval.py | 4 +- rdagent/components/benchmark/eval_method.py | 6 -- rdagent/core/conf.py | 1 + rdagent/oai/llm_utils.py | 69 ++++++++------------- 7 files changed, 69 insertions(+), 72 deletions(-) diff --git a/.env.example b/.env.example index 8b504ad1..9a80a884 100644 --- a/.env.example +++ b/.env.example @@ -1,27 +1,14 @@ # Global configs: -USE_AZURE=True MAX_RETRY=10 RETRY_WAIT_SECONDS=20 -DUMP_CHAT_CACHE=True -USE_CHAT_CACHE=True -DUMP_EMBEDDING_CACHE=True -USE_EMBEDDING_CACHE=True -LOG_LLM_CHAT_CONTENT=False -CHAT_FREQUENCY_PENALTY=0.0 -CHAT_PRESENCE_PENALTY=0.0 -LOG_TRACE_PATH=log_traces + +# api key +OPENAI_API_KEY=your_api_key # embedding model configs: -EMBEDDING_OPENAI_API_KEY=your_api_key -EMBEDDING_AZURE_API_BASE=your_api_base -EMBEDDING_AZURE_API_VERSION=your_api_version EMBEDDING_MODEL=text-embedding-3-small - # chat model configs: -CHAT_OPENAI_API_KEY=your_api_key # 5c -CHAT_AZURE_API_BASE=your_api_base -CHAT_AZURE_API_VERSION=your_api_version CHAT_MODEL=your_model_version CHAT_MAX_TOKENS=3000 CHAT_TEMPERATURE=0.7 diff --git a/README.md b/README.md index 595fc7bd..5efccee5 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,8 @@ TODO: use docker in quick start intead. ### ⚙️ Environment Configuration - Place the `.env` file in the same directory as the `.env.example` file. - - TOOD: please refer to ... for the detailed explanation of the `.env` - - TODO: simplify `.env.example` only keep OpenAI or Azure Azure OpenAI + - The `.env.example` file contains the environment variables required for users using the OpenAI API (Please note that `.env.example` is an example file. `.env` is the one that will be finally used.) + - please refer to [Configuration](docs/build/html/installation.html#azure-openai) for the detailed explanation of the `.env` - Export each variable in the `.env` file: ```sh export $(grep -v '^#' .env | xargs) diff --git a/docs/installation.rst b/docs/installation.rst index d1adfabb..25725ad3 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -12,20 +12,50 @@ For different scenarios Configuration ============= -Quick configuration +To run the application, please create a `.env` file in the root directory of the project and add environment variables according to your requirements. +The standard configuration options for the user using the OpenAI API are provided in the `.env.example` file. +Here are some other configuration options that you can use: +OpenAI API +------------ + +You can use different OpenAI API keys for embedding model and chat model. + + .. code-block:: Properties + + EMBEDDING_OPENAI_API_KEY= + EMBEDDING_MODEL=text-embedding-3-small + + CHAT_OPENAI_API_KEY= + CHAT_MODEL=gpt-4-turbo Azure OpenAI ------------ +The following environment variables are standard configuration options for the user using the OpenAI API. + .. code-block:: Properties + + USE_AZURE=True -USE_AZURE_TOKEN_PROVIDER -~~~~~~~~~~~~~~~~~~~~~~~~ + OPENAI_API_KEY= + + EMBEDDING_MODEL=text-embedding-3-small + EMBEDDING_AZURE_API_BASE= # The base URL for the Azure OpenAI API. + EMBEDDING_AZURE_API_VERSION = # The version of the Azure OpenAI API. -### ☁️ Azure Configuration + CHAT_MODEL=gpt-4-turbo + CHAT_AZURE_API_VERSION = # The version of the Azure OpenAI API. + +Use Azure Token Provider +------------------------ + +If you are using the Azure token provider, you need to set the `USE_AZURE_TOKEN_PROVIDER` environment variable to `True`. then +use the environment variables provided in the `Azure Configuration section `_. + +☁️ Azure Configuration - Install Azure CLI: ```sh diff --git a/rdagent/app/quant_factor_benchmark/eval.py b/rdagent/app/quant_factor_benchmark/eval.py index e98b6a2f..dbae78c7 100644 --- a/rdagent/app/quant_factor_benchmark/eval.py +++ b/rdagent/app/quant_factor_benchmark/eval.py @@ -4,7 +4,7 @@ import time from pathlib import Path from pprint import pprint -from rdagent.app.qlib_rd_loop.conf import PROP_SETTING +from rdagent.app.qlib_rd_loop.conf import FACTOR_PROP_SETTING from rdagent.components.benchmark.conf import BenchmarkSettings from rdagent.components.benchmark.eval_method import FactorImplementEval from rdagent.core.scenario import Scenario @@ -23,7 +23,7 @@ test_cases = FactorTestCaseLoaderFromJsonFile().load(bs.bench_data_path) # 3.declare the method to be tested and pass the arguments. -scen: Scenario = import_class(PROP_SETTING.factor_scen)() +scen: Scenario = import_class(FACTOR_PROP_SETTING.scen)() generate_method = import_class(bs.bench_method_cls)(scen=scen) # 4.declare the eval method and pass the arguments. eval_method = FactorImplementEval( diff --git a/rdagent/components/benchmark/eval_method.py b/rdagent/components/benchmark/eval_method.py index bb5fd817..c94a0e3e 100644 --- a/rdagent/components/benchmark/eval_method.py +++ b/rdagent/components/benchmark/eval_method.py @@ -30,12 +30,6 @@ EVAL_RES = Dict[ ] -EVAL_RES = Dict[ - str, - List[Tuple[FactorEvaluator, Union[object, RunnerException]]], -] - - class TestCase: def __init__( self, diff --git a/rdagent/core/conf.py b/rdagent/core/conf.py index ea152e38..975cad47 100644 --- a/rdagent/core/conf.py +++ b/rdagent/core/conf.py @@ -34,6 +34,7 @@ class RDAgentSettings(BaseSettings): max_past_message_include: int = 10 # Chat configs + openai_api_key: str = "" # TODO: simplify the key design. chat_openai_api_key: str = "" chat_azure_api_base: str = "" chat_azure_api_version: str = "" diff --git a/rdagent/oai/llm_utils.py b/rdagent/oai/llm_utils.py index 75e41928..fdd30a19 100644 --- a/rdagent/oai/llm_utils.py +++ b/rdagent/oai/llm_utils.py @@ -298,7 +298,14 @@ class APIBackend: self.use_azure_token_provider = self.cfg.use_azure_token_provider self.managed_identity_client_id = self.cfg.managed_identity_client_id - self.chat_api_key = self.cfg.chat_openai_api_key if chat_api_key is None else chat_api_key + if self.cfg.openai_api_key: + self.chat_api_key = self.cfg.openai_api_key + self.embedding_api_key = self.cfg.openai_api_key + else: + self.chat_api_key = self.cfg.chat_openai_api_key if chat_api_key is None else chat_api_key + self.embedding_api_key = ( + self.cfg.embedding_openai_api_key if embedding_api_key is None else embedding_api_key + ) self.chat_model = self.cfg.chat_model if chat_model is None else chat_model self.encoder = tiktoken.encoding_for_model(self.chat_model) self.chat_api_base = self.cfg.chat_azure_api_base if chat_api_base is None else chat_api_base @@ -306,9 +313,6 @@ class APIBackend: self.chat_stream = self.cfg.chat_stream self.chat_seed = self.cfg.chat_seed - self.embedding_api_key = ( - self.cfg.embedding_openai_api_key if embedding_api_key is None else embedding_api_key - ) self.embedding_model = self.cfg.embedding_model if embedding_model is None else embedding_model self.embedding_api_base = ( self.cfg.embedding_azure_api_base if embedding_api_base is None else embedding_api_base @@ -610,44 +614,25 @@ class APIBackend: if self.cfg.log_llm_chat_content: logger.info(f"{LogColors.CYAN}Response:{resp}{LogColors.END}", tag="llm_messages") else: - if self.use_azure: - if json_mode: - if add_json_in_prompt: - for message in messages[::-1]: - message["content"] = message["content"] + "\nPlease respond in json format." - if message["role"] == "system": - break - response = self.chat_client.chat.completions.create( - model=self.chat_model, - messages=messages, - max_tokens=max_tokens, - temperature=temperature, - response_format={"type": "json_object"}, - stream=self.chat_stream, - seed=self.chat_seed, - frequency_penalty=frequency_penalty, - presence_penalty=presence_penalty, - ) - else: - response = self.chat_client.chat.completions.create( - model=self.chat_model, - messages=messages, - max_tokens=max_tokens, - temperature=temperature, - stream=self.chat_stream, - seed=self.chat_seed, - frequency_penalty=frequency_penalty, - presence_penalty=presence_penalty, - ) - else: - response = self.chat_client.chat.completions.create( - model=self.chat_model, - messages=messages, - stream=self.chat_stream, - seed=self.chat_seed, - frequency_penalty=frequency_penalty, - presence_penalty=presence_penalty, - ) + kwargs = dict( + model=self.chat_model, + messages=messages, + max_tokens=max_tokens, + temperature=temperature, + stream=self.chat_stream, + seed=self.chat_seed, + frequency_penalty=frequency_penalty, + presence_penalty=presence_penalty, + ) + if json_mode: + if add_json_in_prompt: + for message in messages[::-1]: + message["content"] = message["content"] + "\nPlease respond in json format." + if message["role"] == "system": + break + kwargs["response_format"] = {"type": "json_object"} + response = self.chat_client.chat.completions.create(**kwargs) + if self.chat_stream: resp = "" # TODO: with logger.config(stream=self.chat_stream): and add a `stream_start` flag to add timestamp for first message.