gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 23:47:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Openai api & eval debug (#124)

Browse Source 
* Openai api & eval debug


---------

Co-authored-by: Young <afe.young@gmail.com>
Co-authored-by: you-n-g <you-n-g@users.noreply.github.com>
This commit is contained in:
cyncyw
2024-07-30 18:06:48 +08:00
committed by GitHub
parent dbbec2ffaf
commit 6e8f912968
7 changed files with 69 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+3 -16
View File
@@ -1,27 +1,14 @@
# Global configs:
USE_AZURE=True
MAX_RETRY=10
RETRY_WAIT_SECONDS=20
DUMP_CHAT_CACHE=True
USE_CHAT_CACHE=True
DUMP_EMBEDDING_CACHE=True
USE_EMBEDDING_CACHE=True
LOG_LLM_CHAT_CONTENT=False
CHAT_FREQUENCY_PENALTY=0.0
CHAT_PRESENCE_PENALTY=0.0
LOG_TRACE_PATH=log_traces
# api key
OPENAI_API_KEY=your_api_key
# embedding model configs:
EMBEDDING_OPENAI_API_KEY=your_api_key
EMBEDDING_AZURE_API_BASE=your_api_base
EMBEDDING_AZURE_API_VERSION=your_api_version
EMBEDDING_MODEL=text-embedding-3-small
# chat model configs:
CHAT_OPENAI_API_KEY=your_api_key # 5c
CHAT_AZURE_API_BASE=your_api_base
CHAT_AZURE_API_VERSION=your_api_version
CHAT_MODEL=your_model_version
CHAT_MAX_TOKENS=3000
CHAT_TEMPERATURE=0.7
README.md
+2 -2
View File
@@ -60,8 +60,8 @@ TODO: use docker in quick start intead.
### ⚙️ Environment Configuration
- Place the `.env` file in the same directory as the `.env.example` file.
- TOOD: please refer to ... for the detailed explanation of the `.env`
- TODO: simplify `.env.example` only keep OpenAI or Azure Azure OpenAI
- The `.env.example` file contains the environment variables required for users using the OpenAI API (Please note that `.env.example` is an example file. `.env` is the one that will be finally used.)
- please refer to [Configuration](docs/build/html/installation.html#azure-openai) for the detailed explanation of the `.env`
- Export each variable in the `.env` file:
```sh
export $(grep -v '^#' .env | xargs)
docs/installation.rst
+34 -4
View File
@@ -12,20 +12,50 @@ For different scenarios
Configuration
=============
Quick configuration
To run the application, please create a `.env` file in the root directory of the project and add environment variables according to your requirements.
The standard configuration options for the user using the OpenAI API are provided in the `.env.example` file.
Here are some other configuration options that you can use:
OpenAI API
------------
You can use different OpenAI API keys for embedding model and chat model.
.. code-block:: Properties
EMBEDDING_OPENAI_API_KEY=<replace_with_your_openai_api_key>
EMBEDDING_MODEL=text-embedding-3-small
CHAT_OPENAI_API_KEY=<replace_with_your_openai_api_key>
CHAT_MODEL=gpt-4-turbo
Azure OpenAI
------------
The following environment variables are standard configuration options for the user using the OpenAI API.
.. code-block:: Properties
USE_AZURE=True
USE_AZURE_TOKEN_PROVIDER
~~~~~~~~~~~~~~~~~~~~~~~~
OPENAI_API_KEY=<replace_with_your_openai_api_key>
EMBEDDING_MODEL=text-embedding-3-small
EMBEDDING_AZURE_API_BASE= # The base URL for the Azure OpenAI API.
EMBEDDING_AZURE_API_VERSION = # The version of the Azure OpenAI API.
### ☁️ Azure Configuration
CHAT_MODEL=gpt-4-turbo
CHAT_AZURE_API_VERSION = # The version of the Azure OpenAI API.
Use Azure Token Provider
------------------------
If you are using the Azure token provider, you need to set the `USE_AZURE_TOKEN_PROVIDER` environment variable to `True`. then
use the environment variables provided in the `Azure Configuration section <installation.html#azure-openai>`_.
☁️ Azure Configuration
- Install Azure CLI:
```sh
rdagent/app/quant_factor_benchmark/eval.py
+2 -2
View File
@@ -4,7 +4,7 @@ import time
from pathlib import Path
from pprint import pprint
from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
from rdagent.app.qlib_rd_loop.conf import FACTOR_PROP_SETTING
from rdagent.components.benchmark.conf import BenchmarkSettings
from rdagent.components.benchmark.eval_method import FactorImplementEval
from rdagent.core.scenario import Scenario
@@ -23,7 +23,7 @@ test_cases = FactorTestCaseLoaderFromJsonFile().load(bs.bench_data_path)
# 3.declare the method to be tested and pass the arguments.
scen: Scenario = import_class(PROP_SETTING.factor_scen)()
scen: Scenario = import_class(FACTOR_PROP_SETTING.scen)()
generate_method = import_class(bs.bench_method_cls)(scen=scen)
# 4.declare the eval method and pass the arguments.
eval_method = FactorImplementEval(
rdagent/components/benchmark/eval_method.py
-6
View File
@@ -30,12 +30,6 @@ EVAL_RES = Dict[
]
EVAL_RES = Dict[
str,
List[Tuple[FactorEvaluator, Union[object, RunnerException]]],
]
class TestCase:
def __init__(
self,
rdagent/core/conf.py
+1
View File
@@ -34,6 +34,7 @@ class RDAgentSettings(BaseSettings):
max_past_message_include: int = 10
# Chat configs
openai_api_key: str = "" # TODO: simplify the key design.
chat_openai_api_key: str = ""
chat_azure_api_base: str = ""
chat_azure_api_version: str = ""
rdagent/oai/llm_utils.py
+27 -42
View File
@@ -298,7 +298,14 @@ class APIBackend:
self.use_azure_token_provider = self.cfg.use_azure_token_provider
self.managed_identity_client_id = self.cfg.managed_identity_client_id
self.chat_api_key = self.cfg.chat_openai_api_key if chat_api_key is None else chat_api_key
if self.cfg.openai_api_key:
self.chat_api_key = self.cfg.openai_api_key
self.embedding_api_key = self.cfg.openai_api_key
else:
self.chat_api_key = self.cfg.chat_openai_api_key if chat_api_key is None else chat_api_key
self.embedding_api_key = (
self.cfg.embedding_openai_api_key if embedding_api_key is None else embedding_api_key
)
self.chat_model = self.cfg.chat_model if chat_model is None else chat_model
self.encoder = tiktoken.encoding_for_model(self.chat_model)
self.chat_api_base = self.cfg.chat_azure_api_base if chat_api_base is None else chat_api_base
@@ -306,9 +313,6 @@ class APIBackend:
self.chat_stream = self.cfg.chat_stream
self.chat_seed = self.cfg.chat_seed
self.embedding_api_key = (
self.cfg.embedding_openai_api_key if embedding_api_key is None else embedding_api_key
)
self.embedding_model = self.cfg.embedding_model if embedding_model is None else embedding_model
self.embedding_api_base = (
self.cfg.embedding_azure_api_base if embedding_api_base is None else embedding_api_base
@@ -610,44 +614,25 @@ class APIBackend:
if self.cfg.log_llm_chat_content:
logger.info(f"{LogColors.CYAN}Response:{resp}{LogColors.END}", tag="llm_messages")
else:
if self.use_azure:
if json_mode:
if add_json_in_prompt:
for message in messages[::-1]:
message["content"] = message["content"] + "\nPlease respond in json format."
if message["role"] == "system":
break
response = self.chat_client.chat.completions.create(
model=self.chat_model,
messages=messages,
max_tokens=max_tokens,
temperature=temperature,
response_format={"type": "json_object"},
stream=self.chat_stream,
seed=self.chat_seed,
frequency_penalty=frequency_penalty,
presence_penalty=presence_penalty,
)
else:
response = self.chat_client.chat.completions.create(
model=self.chat_model,
messages=messages,
max_tokens=max_tokens,
temperature=temperature,
stream=self.chat_stream,
seed=self.chat_seed,
frequency_penalty=frequency_penalty,
presence_penalty=presence_penalty,
)
else:
response = self.chat_client.chat.completions.create(
model=self.chat_model,
messages=messages,
stream=self.chat_stream,
seed=self.chat_seed,
frequency_penalty=frequency_penalty,
presence_penalty=presence_penalty,
)
kwargs = dict(
model=self.chat_model,
messages=messages,
max_tokens=max_tokens,
temperature=temperature,
stream=self.chat_stream,
seed=self.chat_seed,
frequency_penalty=frequency_penalty,
presence_penalty=presence_penalty,
)
if json_mode:
if add_json_in_prompt:
for message in messages[::-1]:
message["content"] = message["content"] + "\nPlease respond in json format."
if message["role"] == "system":
break
kwargs["response_format"] = {"type": "json_object"}
response = self.chat_client.chat.completions.create(**kwargs)
if self.chat_stream:
resp = ""
# TODO: with logger.config(stream=self.chat_stream): and add a `stream_start` flag to add timestamp for first message.