gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-08-04 10:47:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

use token limit to shrink report content (#24)

Browse Source 
Co-authored-by: xuyang1 <xuyang1@microsoft.com>
This commit is contained in:
Xu Yang
2024-06-13 16:20:54 +08:00
committed by GitHub
parent 55959c93ff
commit 9e82da243b
4 changed files with 30 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rdagent/document_process/document_analysis.py
+11 -6
View File
@@ -69,14 +69,17 @@ def classify_report_from_dict(
res_dict[file_name] = {"class": 0}
continue
if not any(substring in content for substring in substrings):
if not any(substring in content for substring in substrings) and False:
res_dict[file_name] = {"class": 0}
else:
gpt_4_max_token = 128000
if input_max_token < gpt_4_max_token:
content = enc.encode(content)
max_token_1 = max(0, min(len(content), input_max_token) - 1)
content = enc.decode(content[:max_token_1])
while (
APIBackend().build_messages_and_calculate_token(
user_prompt=content,
system_prompt=classify_prompt,
)
> Config().chat_token_limit
):
content = content[: -(Config().chat_token_limit // 100)]
vote_list = []
for _ in range(vote_time):
@@ -450,6 +453,8 @@ def __kmeans_embeddings(embeddings: np.ndarray, k: int = 20) -> list[list[str]]:
def __deduplicate_factor_dict(factor_dict: dict[str, dict[str, str]]) -> list[list[str]]:
if len(factor_dict) == 0:
return []
factor_df = pd.DataFrame(factor_dict).T
factor_df.index.names = ["factor_name"]