mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-31 17:27:42 +00:00
Refine all the implementation code to higher quality for release (#29)
* refine CI script * refine all the code to higher quality * refine the script to factor extraction and implementation * add task loader interface * add a task loader interface && move pdf analysis to pdf task loader * change the name to global variables --------- Co-authored-by: xuyang1 <xuyang1@microsoft.com>
This commit is contained in:
Xu Yang
@@ -0,0 +1,31 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
from rdagent.core.task import TaskLoader
|
||||
from rdagent.factor_implementation.evolving.factor import FactorImplementTask
|
||||
|
||||
|
||||
class FactorImplementationTaskLoaderFromDict(TaskLoader):
|
||||
def load(self, factor_dict: dict) -> list:
|
||||
"""Load data from a dict."""
|
||||
task_l = []
|
||||
for factor_name, factor_data in factor_dict.items():
|
||||
task = FactorImplementTask(
|
||||
factor_name=factor_name,
|
||||
factor_description=factor_data["description"],
|
||||
factor_formulation=factor_data["formulation"],
|
||||
variables=factor_data["variables"],
|
||||
)
|
||||
task_l.append(task)
|
||||
return task_l
|
||||
|
||||
|
||||
class FactorImplementationTaskLoaderFromJsonFile(TaskLoader):
|
||||
def load(self, json_file_path: Path) -> list:
|
||||
factor_dict = json.load(json_file_path)
|
||||
return FactorImplementationTaskLoaderFromDict().load(factor_dict)
|
||||
|
||||
|
||||
class FactorImplementationTaskLoaderFromJsonString(TaskLoader):
|
||||
def load(self, json_string: str) -> list:
|
||||
factor_dict = json.loads(json_string)
|
||||
return FactorImplementationTaskLoaderFromDict().load(factor_dict)
|
||||
@@ -0,0 +1,584 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import multiprocessing as mp
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Mapping
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import tiktoken
|
||||
from jinja2 import Template
|
||||
from rdagent.core.conf import RD_AGENT_SETTINGS
|
||||
from rdagent.core.log import RDAgentLog
|
||||
from rdagent.core.prompts import Prompts
|
||||
from rdagent.core.task import TaskLoader
|
||||
from rdagent.document_reader.document_reader import load_and_process_pdfs_by_langchain
|
||||
from rdagent.factor_implementation.task_loader.json_loader import FactorImplementationTaskLoaderFromDict
|
||||
from rdagent.oai.llm_utils import APIBackend, create_embedding_with_multiprocessing
|
||||
from sklearn.cluster import KMeans
|
||||
from sklearn.metrics.pairwise import cosine_similarity
|
||||
from sklearn.preprocessing import normalize
|
||||
|
||||
document_process_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
|
||||
|
||||
|
||||
def classify_report_from_dict(
|
||||
report_dict: Mapping[str, str],
|
||||
vote_time: int = 1,
|
||||
substrings: tuple[str] = (),
|
||||
) -> dict[str, dict[str, str]]:
|
||||
"""
|
||||
Parameters:
|
||||
- report_dict (Dict[str, str]):
|
||||
A dictionary where the key is the path of the report (ending with .pdf),
|
||||
and the value is either the report content as a string.
|
||||
- input_max_token (int): Specifying the maximum number of input tokens.
|
||||
- vote_time (int): An integer specifying how many times to vote.
|
||||
- substrings (list(str)): List of hardcode substrings.
|
||||
|
||||
Returns:
|
||||
- Dict[str, Dict[str, str]]: A dictionary where each key is the path of the report,
|
||||
with a single key 'class' and its value being the classification result (0 or 1).
|
||||
|
||||
"""
|
||||
# if len(substrings) == 0:
|
||||
# substrings = (
|
||||
# "金融工程",
|
||||
# "金工",
|
||||
# "回测",
|
||||
# "因子",
|
||||
# "机器学习",
|
||||
# "深度学习",
|
||||
# "量化",
|
||||
# )
|
||||
|
||||
res_dict = {}
|
||||
classify_prompt = document_process_prompts["classify_system"]
|
||||
|
||||
for key, value in report_dict.items():
|
||||
if not key.endswith(".pdf"):
|
||||
continue
|
||||
file_name = key
|
||||
|
||||
if isinstance(value, str):
|
||||
content = value
|
||||
else:
|
||||
RDAgentLog().warning(f"输入格式不符合要求: {file_name}")
|
||||
res_dict[file_name] = {"class": 0}
|
||||
continue
|
||||
|
||||
# pre-filter document with key words is not necessary, skip this check for now
|
||||
# if (
|
||||
# not any(substring in content for substring in substrings) and False
|
||||
# ):
|
||||
# res_dict[file_name] = {"class": 0}
|
||||
# else:
|
||||
while (
|
||||
APIBackend().build_messages_and_calculate_token(
|
||||
user_prompt=content,
|
||||
system_prompt=classify_prompt,
|
||||
)
|
||||
> RD_AGENT_SETTINGS.chat_token_limit
|
||||
):
|
||||
content = content[: -(RD_AGENT_SETTINGS.chat_token_limit // 100)]
|
||||
|
||||
vote_list = []
|
||||
for _ in range(vote_time):
|
||||
user_prompt = content
|
||||
system_prompt = classify_prompt
|
||||
res = APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
json_mode=True,
|
||||
)
|
||||
try:
|
||||
res = json.loads(res)
|
||||
vote_list.append(int(res["class"]))
|
||||
except json.JSONDecodeError:
|
||||
RDAgentLog().warning(f"返回值无法解析: {file_name}")
|
||||
res_dict[file_name] = {"class": 0}
|
||||
count_0 = vote_list.count(0)
|
||||
count_1 = vote_list.count(1)
|
||||
if max(count_0, count_1) > int(vote_time / 2):
|
||||
break
|
||||
|
||||
result = 1 if count_1 > count_0 else 0
|
||||
res_dict[file_name] = {"class": result}
|
||||
|
||||
return res_dict
|
||||
|
||||
|
||||
def __extract_factors_name_and_desc_from_content(
|
||||
content: str,
|
||||
) -> dict[str, dict[str, str]]:
|
||||
session = APIBackend().build_chat_session(
|
||||
session_system_prompt=document_process_prompts["extract_factors_system"],
|
||||
)
|
||||
|
||||
extracted_factor_dict = {}
|
||||
current_user_prompt = content
|
||||
|
||||
for _ in range(10):
|
||||
extract_result_resp = session.build_chat_completion(
|
||||
user_prompt=current_user_prompt,
|
||||
json_mode=False,
|
||||
)
|
||||
re_search_res = re.search(r"```json(.*)```", extract_result_resp, re.S)
|
||||
ret_json_str = re_search_res.group(1) if re_search_res is not None else ""
|
||||
try:
|
||||
ret_dict = json.loads(ret_json_str)
|
||||
parse_success = bool(isinstance(ret_dict, dict)) and "factors" in ret_dict
|
||||
except json.JSONDecodeError:
|
||||
parse_success = False
|
||||
if ret_json_str is None or not parse_success:
|
||||
current_user_prompt = "Your response didn't follow the instruction might be wrong json format. Try again."
|
||||
else:
|
||||
factors = ret_dict["factors"]
|
||||
if len(factors) == 0:
|
||||
break
|
||||
for factor_name, factor_description in factors.items():
|
||||
extracted_factor_dict[factor_name] = factor_description
|
||||
current_user_prompt = document_process_prompts["extract_factors_follow_user"]
|
||||
|
||||
return extracted_factor_dict
|
||||
|
||||
|
||||
def __extract_factors_formulation_from_content(
|
||||
content: str,
|
||||
factor_dict: dict[str, str],
|
||||
) -> dict[str, dict[str, str]]:
|
||||
factor_dict_df = pd.DataFrame(
|
||||
factor_dict.items(),
|
||||
columns=["factor_name", "factor_description"],
|
||||
)
|
||||
|
||||
system_prompt = document_process_prompts["extract_factor_formulation_system"]
|
||||
current_user_prompt = Template(
|
||||
document_process_prompts["extract_factor_formulation_user"],
|
||||
).render(report_content=content, factor_dict=factor_dict_df.to_string())
|
||||
|
||||
session = APIBackend().build_chat_session(session_system_prompt=system_prompt)
|
||||
factor_to_formulation = {}
|
||||
|
||||
for _ in range(10):
|
||||
extract_result_resp = session.build_chat_completion(
|
||||
user_prompt=current_user_prompt,
|
||||
json_mode=False,
|
||||
)
|
||||
re_search_res = re.search(r"```json(.*)```", extract_result_resp, re.S)
|
||||
ret_json_str = re_search_res.group(1) if re_search_res is not None else ""
|
||||
try:
|
||||
ret_dict = json.loads(ret_json_str)
|
||||
parse_success = bool(isinstance(ret_dict, dict))
|
||||
except json.JSONDecodeError:
|
||||
parse_success = False
|
||||
if ret_json_str is None or not parse_success:
|
||||
current_user_prompt = "Your response didn't follow the instruction might be wrong json format. Try again."
|
||||
else:
|
||||
for name, formulation_and_description in ret_dict.items():
|
||||
if name in factor_dict:
|
||||
factor_to_formulation[name] = formulation_and_description
|
||||
if len(factor_to_formulation) != len(factor_dict):
|
||||
remain_df = factor_dict_df[~factor_dict_df["factor_name"].isin(factor_to_formulation)]
|
||||
current_user_prompt = (
|
||||
"Some factors are missing. Please check the following"
|
||||
" factors and their descriptions and continue extraction.\n"
|
||||
"==========================Remaining factors"
|
||||
"==========================\n" + remain_df.to_string()
|
||||
)
|
||||
else:
|
||||
break
|
||||
|
||||
return factor_to_formulation
|
||||
|
||||
|
||||
def __extract_factor_and_formulation_from_one_report(
|
||||
content: str,
|
||||
) -> dict[str, dict[str, str]]:
|
||||
final_factor_dict_to_one_report = {}
|
||||
factor_dict = __extract_factors_name_and_desc_from_content(content)
|
||||
if len(factor_dict) != 0:
|
||||
factor_to_formulation = __extract_factors_formulation_from_content(
|
||||
content,
|
||||
factor_dict,
|
||||
)
|
||||
for factor_name in factor_dict:
|
||||
if factor_name not in factor_to_formulation:
|
||||
continue
|
||||
|
||||
final_factor_dict_to_one_report.setdefault(factor_name, {})
|
||||
final_factor_dict_to_one_report[factor_name]["description"] = factor_dict[factor_name]
|
||||
|
||||
# use code to correct _ in formulation
|
||||
formulation = factor_to_formulation[factor_name]["formulation"]
|
||||
if factor_name in formulation:
|
||||
target_factor_name = factor_name.replace("_", r"\_")
|
||||
formulation = formulation.replace(factor_name, target_factor_name)
|
||||
for variable in factor_to_formulation[factor_name]["variables"]:
|
||||
if variable in formulation:
|
||||
target_variable = variable.replace("_", r"\_")
|
||||
formulation = formulation.replace(variable, target_variable)
|
||||
|
||||
final_factor_dict_to_one_report[factor_name]["formulation"] = formulation
|
||||
final_factor_dict_to_one_report[factor_name]["variables"] = factor_to_formulation[factor_name]["variables"]
|
||||
|
||||
return final_factor_dict_to_one_report
|
||||
|
||||
|
||||
def extract_factors_from_report_dict(
|
||||
report_dict: dict[str, str],
|
||||
useful_no_dict: dict[str, dict[str, str]],
|
||||
n_proc: int = 11,
|
||||
) -> dict[str, dict[str, dict[str, str]]]:
|
||||
useful_report_dict = {}
|
||||
for key, value in useful_no_dict.items():
|
||||
if isinstance(value, dict):
|
||||
if int(value.get("class")) == 1:
|
||||
useful_report_dict[key] = report_dict[key]
|
||||
else:
|
||||
RDAgentLog().warning(f"Invalid input format: {key}")
|
||||
|
||||
final_report_factor_dict = {}
|
||||
# for file_name, content in useful_report_dict.items():
|
||||
# final_report_factor_dict.setdefault(file_name, {})
|
||||
# final_report_factor_dict[file_name] = __extract_factor_and_formulation_from_one_report(content)
|
||||
|
||||
while len(final_report_factor_dict) != len(useful_report_dict):
|
||||
pool = mp.Pool(n_proc)
|
||||
pool_result_list = []
|
||||
file_names = []
|
||||
for file_name, content in useful_report_dict.items():
|
||||
if file_name in final_report_factor_dict:
|
||||
continue
|
||||
file_names.append(file_name)
|
||||
pool_result_list.append(
|
||||
pool.apply_async(
|
||||
__extract_factor_and_formulation_from_one_report,
|
||||
(content,),
|
||||
),
|
||||
)
|
||||
|
||||
pool.close()
|
||||
pool.join()
|
||||
|
||||
for index, result in enumerate(pool_result_list):
|
||||
if result.get is not None:
|
||||
file_name = file_names[index]
|
||||
final_report_factor_dict.setdefault(file_name, {})
|
||||
final_report_factor_dict[file_name] = result.get()
|
||||
RDAgentLog().info(f"已经完成{len(final_report_factor_dict)}个报告的因子提取")
|
||||
|
||||
return final_report_factor_dict
|
||||
|
||||
|
||||
def merge_file_to_factor_dict_to_factor_dict(
|
||||
file_to_factor_dict: dict[str, dict],
|
||||
) -> dict:
|
||||
factor_dict = {}
|
||||
for file_name in file_to_factor_dict:
|
||||
for factor_name in file_to_factor_dict[file_name]:
|
||||
factor_dict.setdefault(factor_name, [])
|
||||
factor_dict[factor_name].append(file_to_factor_dict[file_name][factor_name])
|
||||
|
||||
factor_dict_simple_deduplication = {}
|
||||
for factor_name in factor_dict:
|
||||
if len(factor_dict[factor_name]) > 1:
|
||||
factor_dict_simple_deduplication[factor_name] = max(
|
||||
factor_dict[factor_name],
|
||||
key=lambda x: len(x["formulation"]),
|
||||
)
|
||||
else:
|
||||
factor_dict_simple_deduplication[factor_name] = factor_dict[factor_name][0]
|
||||
return factor_dict_simple_deduplication
|
||||
|
||||
|
||||
def __check_factor_dict_viability_simulate_json_mode(
|
||||
factor_df_string: str,
|
||||
) -> dict[str, dict[str, str]]:
|
||||
session = APIBackend().build_chat_session(
|
||||
session_system_prompt=document_process_prompts["factor_viability_system"],
|
||||
)
|
||||
current_user_prompt = factor_df_string
|
||||
|
||||
for _ in range(10):
|
||||
extract_result_resp = session.build_chat_completion(
|
||||
user_prompt=current_user_prompt,
|
||||
json_mode=False,
|
||||
)
|
||||
re_search_res = re.search(r"```json(.*)```", extract_result_resp, re.S)
|
||||
ret_json_str = re_search_res.group(1) if re_search_res is not None else ""
|
||||
try:
|
||||
ret_dict = json.loads(ret_json_str)
|
||||
parse_success = bool(isinstance(ret_dict, dict))
|
||||
except json.JSONDecodeError:
|
||||
parse_success = False
|
||||
if ret_json_str is None or not parse_success:
|
||||
current_user_prompt = "Your response didn't follow the instruction might be wrong json format. Try again."
|
||||
else:
|
||||
return ret_dict
|
||||
return {}
|
||||
|
||||
|
||||
def check_factor_viability(
|
||||
factor_dict: dict[str, dict[str, str]],
|
||||
) -> tuple[dict[str, dict[str, str]], dict[str, dict[str, str]]]:
|
||||
factor_viability_dict = {}
|
||||
|
||||
factor_df = pd.DataFrame(factor_dict).T
|
||||
factor_df.index.names = ["factor_name"]
|
||||
|
||||
while factor_df.shape[0] > 0:
|
||||
pool = mp.Pool(8)
|
||||
|
||||
result_list = []
|
||||
for i in range(0, factor_df.shape[0], 50):
|
||||
target_factor_df_string = factor_df.iloc[i : i + 50, :].to_string()
|
||||
|
||||
result_list.append(
|
||||
pool.apply_async(
|
||||
__check_factor_dict_viability_simulate_json_mode,
|
||||
(target_factor_df_string,),
|
||||
),
|
||||
)
|
||||
|
||||
pool.close()
|
||||
pool.join()
|
||||
|
||||
for result in result_list:
|
||||
respond = result.get()
|
||||
for factor_name, viability in respond.items():
|
||||
factor_viability_dict[factor_name] = viability
|
||||
|
||||
factor_df = factor_df[~factor_df.index.isin(factor_viability_dict)]
|
||||
|
||||
# filtered_factor_dict = {
|
||||
# factor_name: factor_dict[factor_name]
|
||||
# for factor_name in factor_dict
|
||||
# if factor_viability_dict[factor_name]["viability"]
|
||||
# }
|
||||
|
||||
return factor_viability_dict
|
||||
|
||||
|
||||
def __check_factor_duplication_simulate_json_mode(
|
||||
factor_df: pd.DataFrame,
|
||||
) -> list[list[str]]:
|
||||
session = APIBackend().build_chat_session(
|
||||
session_system_prompt=document_process_prompts["factor_duplicate_system"],
|
||||
)
|
||||
current_user_prompt = factor_df.to_string()
|
||||
|
||||
generated_duplicated_groups = []
|
||||
for _ in range(20):
|
||||
extract_result_resp = session.build_chat_completion(
|
||||
user_prompt=current_user_prompt,
|
||||
json_mode=False,
|
||||
)
|
||||
re_search_res = re.search(r"```json(.*)```", extract_result_resp, re.S)
|
||||
ret_json_str = re_search_res.group(1) if re_search_res is not None else ""
|
||||
try:
|
||||
ret_dict = json.loads(ret_json_str)
|
||||
parse_success = bool(isinstance(ret_dict, list))
|
||||
except json.JSONDecodeError:
|
||||
parse_success = False
|
||||
if ret_json_str is None or not parse_success:
|
||||
current_user_prompt = (
|
||||
"Your previous response didn't follow"
|
||||
" the instruction might be wrong json"
|
||||
" format. Try reducing the factors."
|
||||
)
|
||||
elif len(ret_dict) == 0:
|
||||
return generated_duplicated_groups
|
||||
else:
|
||||
generated_duplicated_groups.extend(ret_dict)
|
||||
current_user_prompt = (
|
||||
"Continue to extract duplicated"
|
||||
" groups. If no more duplicated group"
|
||||
" found please respond empty dict."
|
||||
)
|
||||
return generated_duplicated_groups
|
||||
|
||||
|
||||
def __kmeans_embeddings(embeddings: np.ndarray, k: int = 20) -> list[list[str]]:
|
||||
x_normalized = normalize(embeddings)
|
||||
|
||||
kmeans = KMeans(
|
||||
n_clusters=k,
|
||||
init="random",
|
||||
max_iter=100,
|
||||
n_init=10,
|
||||
random_state=42,
|
||||
)
|
||||
|
||||
# KMeans algorithm uses Euclidean distance, and we need to customize a function to find the most similar cluster center
|
||||
def find_closest_cluster_cosine_similarity(
|
||||
data: np.ndarray,
|
||||
centroids: np.ndarray,
|
||||
) -> np.ndarray:
|
||||
similarity = cosine_similarity(data, centroids)
|
||||
return np.argmax(similarity, axis=1)
|
||||
|
||||
# Initializes the cluster center
|
||||
rng = np.random.default_rng()
|
||||
centroids = rng.choice(x_normalized, size=k, replace=False)
|
||||
|
||||
# Iterate until convergence or the maximum number of iterations is reached
|
||||
for _ in range(kmeans.max_iter):
|
||||
# Assign the sample to the nearest cluster center
|
||||
closest_clusters = find_closest_cluster_cosine_similarity(
|
||||
x_normalized,
|
||||
centroids,
|
||||
)
|
||||
|
||||
# update the cluster center
|
||||
new_centroids = np.array(
|
||||
[x_normalized[closest_clusters == i].mean(axis=0) for i in range(k)],
|
||||
)
|
||||
new_centroids = normalize(new_centroids) # 归一化新的簇中心
|
||||
|
||||
# Check whether the cluster center has changed
|
||||
if np.allclose(centroids, new_centroids):
|
||||
break
|
||||
|
||||
centroids = new_centroids
|
||||
|
||||
clusters = find_closest_cluster_cosine_similarity(x_normalized, centroids)
|
||||
cluster_to_index = {}
|
||||
for index, cluster in enumerate(clusters):
|
||||
cluster_to_index.setdefault(cluster, []).append(index)
|
||||
return sorted(
|
||||
cluster_to_index.values(),
|
||||
key=lambda x: len(x),
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
|
||||
def __deduplicate_factor_dict(factor_dict: dict[str, dict[str, str]]) -> list[list[str]]:
|
||||
if len(factor_dict) == 0:
|
||||
return []
|
||||
factor_df = pd.DataFrame(factor_dict).T
|
||||
factor_df.index.names = ["factor_name"]
|
||||
|
||||
factor_names = sorted(factor_dict)
|
||||
|
||||
factor_name_to_full_str = {}
|
||||
for factor_name in factor_dict:
|
||||
description = factor_dict[factor_name]["description"]
|
||||
formulation = factor_dict[factor_name]["formulation"]
|
||||
variables = factor_dict[factor_name]["variables"]
|
||||
factor_name_to_full_str[
|
||||
factor_name
|
||||
] = f"""Factor name: {factor_name}
|
||||
Factor description: {description}
|
||||
Factor formulation: {formulation}
|
||||
Factor variables: {variables}
|
||||
"""
|
||||
|
||||
full_str_list = [factor_name_to_full_str[factor_name] for factor_name in factor_names]
|
||||
embeddings = create_embedding_with_multiprocessing(full_str_list)
|
||||
|
||||
target_k = None
|
||||
if len(full_str_list) < RD_AGENT_SETTINGS.max_input_duplicate_factor_group:
|
||||
kmeans_index_group = [list(range(len(full_str_list)))]
|
||||
target_k = 1
|
||||
else:
|
||||
for k in range(
|
||||
len(full_str_list) // RD_AGENT_SETTINGS.max_input_duplicate_factor_group,
|
||||
30,
|
||||
):
|
||||
kmeans_index_group = __kmeans_embeddings(embeddings=embeddings, k=k)
|
||||
if len(kmeans_index_group[0]) < RD_AGENT_SETTINGS.max_input_duplicate_factor_group:
|
||||
target_k = k
|
||||
RDAgentLog().info(f"K-means group number: {k}")
|
||||
break
|
||||
factor_name_groups = [[factor_names[index] for index in index_group] for index_group in kmeans_index_group]
|
||||
|
||||
duplication_names_list = []
|
||||
|
||||
pool = mp.Pool(target_k)
|
||||
result_list = []
|
||||
result_list = [
|
||||
pool.apply_async(
|
||||
__check_factor_duplication_simulate_json_mode,
|
||||
(factor_df.loc[factor_name_group, :],),
|
||||
)
|
||||
for factor_name_group in factor_name_groups
|
||||
]
|
||||
|
||||
pool.close()
|
||||
pool.join()
|
||||
|
||||
for result in result_list:
|
||||
deduplication_factor_names_list = result.get()
|
||||
for deduplication_factor_names in deduplication_factor_names_list:
|
||||
filter_factor_names = [
|
||||
factor_name for factor_name in set(deduplication_factor_names) if factor_name in factor_dict
|
||||
]
|
||||
if len(filter_factor_names) > 1:
|
||||
duplication_names_list.append(filter_factor_names)
|
||||
|
||||
return duplication_names_list
|
||||
|
||||
|
||||
def deduplicate_factors_by_llm( # noqa: C901, PLR0912
|
||||
factor_dict: dict[str, dict[str, str]],
|
||||
factor_viability_dict: dict[str, dict[str, str]] | None = None,
|
||||
) -> list[list[str]]:
|
||||
final_duplication_names_list = []
|
||||
current_round_factor_dict = factor_dict
|
||||
for _ in range(10):
|
||||
duplication_names_list = __deduplicate_factor_dict(current_round_factor_dict)
|
||||
|
||||
new_round_names = []
|
||||
for duplication_names in duplication_names_list:
|
||||
if len(duplication_names) < RD_AGENT_SETTINGS.max_output_duplicate_factor_group:
|
||||
final_duplication_names_list.append(duplication_names)
|
||||
else:
|
||||
new_round_names.extend(duplication_names)
|
||||
|
||||
if len(new_round_names) != 0:
|
||||
current_round_factor_dict = {factor_name: factor_dict[factor_name] for factor_name in new_round_names}
|
||||
else:
|
||||
break
|
||||
|
||||
final_duplication_names_list = sorted(final_duplication_names_list, key=lambda x: len(x), reverse=True)
|
||||
|
||||
to_replace_dict = {}
|
||||
for duplication_names in duplication_names_list:
|
||||
if factor_viability_dict is not None:
|
||||
viability_list = [factor_viability_dict[name]["viability"] for name in duplication_names]
|
||||
if True not in viability_list:
|
||||
continue
|
||||
target_factor_name = duplication_names[viability_list.index(True)]
|
||||
else:
|
||||
target_factor_name = duplication_names[0]
|
||||
for duplication_factor_name in duplication_names:
|
||||
if duplication_factor_name == target_factor_name:
|
||||
continue
|
||||
to_replace_dict[duplication_factor_name] = target_factor_name
|
||||
|
||||
llm_deduplicated_factor_dict = {}
|
||||
added_lower_name_set = set()
|
||||
for factor_name in factor_dict:
|
||||
if factor_name not in to_replace_dict and factor_name.lower() not in added_lower_name_set:
|
||||
if factor_viability_dict is not None and not factor_viability_dict[factor_name]["viability"]:
|
||||
continue
|
||||
added_lower_name_set.add(factor_name.lower())
|
||||
llm_deduplicated_factor_dict[factor_name] = factor_dict[factor_name]
|
||||
|
||||
return llm_deduplicated_factor_dict, final_duplication_names_list
|
||||
|
||||
|
||||
class FactorImplementationTaskLoaderFromPDFfiles(TaskLoader):
|
||||
def load(self, file_or_folder_path: Path) -> dict:
|
||||
docs_dict = load_and_process_pdfs_by_langchain(Path(file_or_folder_path))
|
||||
|
||||
selected_report_dict = classify_report_from_dict(report_dict=docs_dict, vote_time=1)
|
||||
file_to_factor_result = extract_factors_from_report_dict(docs_dict, selected_report_dict)
|
||||
factor_dict = merge_file_to_factor_dict_to_factor_dict(file_to_factor_result)
|
||||
|
||||
factor_viability = check_factor_viability(factor_dict)
|
||||
factor_dict, duplication_names_list = deduplicate_factors_by_llm(factor_dict, factor_viability)
|
||||
return FactorImplementationTaskLoaderFromDict().load(factor_dict)
|
||||
@@ -0,0 +1,201 @@
|
||||
extract_factors_system: |-
|
||||
用户会提供一篇金融工程研报，其中包括了量化因子和模型研究，请按照要求抽取以下信息:
|
||||
1. 概述这篇研报的主要研究思路;
|
||||
2. 抽取出所有的因子，并概述因子的计算过程，请注意有些因子可能存在于表格中，请不要遗漏，因子的名称请使用英文，不能包含空格，可用下划线连接，研报中可能不含有因子，若没有请返回空字典;
|
||||
3. 抽取研报里面的所有模型，并概述模型的计算过程，可以分步骤描述模型搭建或计算的过程，研报中可能不含有模型，若没有请返回空字典;
|
||||
|
||||
user will treat your factor name as key to store the factor, don't put any interaction message in the content. Just response the output without any interaction and explanation.
|
||||
All names should be in English.
|
||||
Respond with your analysis in JSON format. The JSON schema should include:
|
||||
```json
|
||||
{
|
||||
"summary": "The summary of this report",
|
||||
"factors": {
|
||||
"Name of factor 1": "Description to factor 1",
|
||||
"Name of factor 2": "Description to factor 2"
|
||||
},
|
||||
"models": {
|
||||
"Name of model 1": "Description to model 1",
|
||||
"Name of model 2": "Description to model 2"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
extract_factors_follow_user: |-
|
||||
Please continue extracting the factors. Please ignore factors appeared in former messages. If no factor is found, please return an empty dict.
|
||||
Notice: You should not miss any factor in the report! Some factors might appear several times in the report. You can repeat them to avoid missing other factors.
|
||||
Respond with your analysis in JSON format. The JSON schema should include:
|
||||
```json
|
||||
{
|
||||
"factors": {
|
||||
"Name of factor 1": "Description to factor 1",
|
||||
"Name of factor 2": "Description to factor 2"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
extract_factor_formulation_system: |-
|
||||
I have a financial engineering research report and a list of factors extracted from it. I need assistance in extracting specific information based on the report and the provided list of factors. The tasks are as follows:
|
||||
|
||||
1. For each factor, I need its calculation formula in LaTeX format. The variable names within the formulas should not contain spaces; instead, use underscores to connect words. Ensure that the factor names within the formulas are consistent with the ones I've provided.
|
||||
2. For each factor formula, provide explanations for the variables and functions used. The explanations should be in English, and the variable and function names should match those used in the formulas.
|
||||
|
||||
Here are the sources of data I have:
|
||||
|
||||
1. Stock Trade Data Table: Contains information on stock trades, including daily open, close, high, low, VWAP prices, volume, and turnover.
|
||||
2. Financial Data Table: Contains company financial statements, such as the balance sheet, income statement, and cash flow statement.
|
||||
3. Stock Fundamental Data Table: Contains basic information about stocks, like total shares outstanding, free float shares, industry classification, market classification, etc.
|
||||
4. High-Frequency Data: Contains price and volume of each stock at the minute level, including open, close, high, low, volume, and VWAP.
|
||||
|
||||
Please expand the formulation to use the source data I have provided. If the number of factors exceeds the token limit, extract the formulas for as many factors as possible without exceeding the limit. Ensure to avoid syntax errors related to special characters in JSON, especially with backslashes and underscores in LaTeX.
|
||||
|
||||
Provide your analysis in JSON format, using the following schema:
|
||||
|
||||
```json
|
||||
{
|
||||
"factor name 1": {
|
||||
"formulation": "latex formulation of factor 1",
|
||||
"variables": {
|
||||
"variable or function name 1": "description of variable or function 1",
|
||||
"variable or function name 2": "description of variable or function 2"
|
||||
}
|
||||
},
|
||||
"factor name 2": {
|
||||
"formulation": "latex formulation of factor 2",
|
||||
"variables": {
|
||||
"variable or function name 1": "description of variable or function 1",
|
||||
"variable or function name 2": "description of variable or function 2"
|
||||
}
|
||||
}
|
||||
# Don't add ellipsis (...) or any filler text that might cause JSON parsing errors here!
|
||||
}
|
||||
```
|
||||
|
||||
Please only include the factors you've extracted in your response, without any additional messages or explanations. If you can only provide part of the factors, do not use ellipsis or any filler text that might cause JSON parsing errors.
|
||||
|
||||
|
||||
extract_factor_formulation_user: |-
|
||||
===========================Report content:=============================
|
||||
{{ report_content }}
|
||||
===========================Factor list in dataframe=============================
|
||||
{{ factor_dict }}
|
||||
|
||||
classify_system_chinese: |-
|
||||
你是一个研报分类助手。用户会输入一篇金融研报。请按照要求回答：
|
||||
因子指能够解释资产收益率或价格等的变量；而模型则指机器学习或深度学习模型，利用因子等变量来预测价格或收益率变化。
|
||||
|
||||
请你对研报进行分类，考虑两个条件：
|
||||
1. 是金工量化领域中选股（需与择时，选基等严格区分开）方面的研报;
|
||||
2. 涉及了因子或模型的构成，或者是测试了它们的表现。
|
||||
如果研报同时满足上述两个条件，请输出1；若没有，请输出0。
|
||||
|
||||
请使用json进行回答。json key为：class
|
||||
|
||||
classify_system: |-
|
||||
Your job is classify whether the user input document is a quantitative investment research report. The user will input a document and you should classify it based on the following conditions:
|
||||
1. The document is about finance other than other fields like biology, physics, chemistry, etc.
|
||||
2. The document is a research report on stock selection (which needs to be strictly separated from time selection and base selection) in the field of metalworking quantification.
|
||||
3. The document involves the composition of factors or models, or tests their performance.
|
||||
|
||||
If the document meets all the above conditions, please return 1; otherwise, please return 0.
|
||||
|
||||
Please respond with your decision in JSON format. Just respond the output json string without any interaction and explanation.
|
||||
The JSON schema should include:
|
||||
{
|
||||
"class": 1
|
||||
}
|
||||
|
||||
factor_viability_system: |-
|
||||
User has designed several factors in quant investment. Please help the user to check the viability of these factors.
|
||||
These factors are used to build a daily frequency strategy in China A-share market.
|
||||
|
||||
User will provide a pandas dataframe like table containing following information:
|
||||
1. The name of the factor;
|
||||
2. The simple description of the factor;
|
||||
3. The formulation of the factor in latex format;
|
||||
4. The description to the variables and functions in the formulation of the factor.
|
||||
|
||||
User has several source data:
|
||||
1. The Stock Trade Data Table containing information about stock trades, such as daily open, close, high, low, vwap prices, volume, and turnover;
|
||||
2. The Financial Data Table containing company financial statements such as the balance sheet, income statement, and cash flow statement;
|
||||
3. The Stock Fundamental Data Table containing basic information about stocks, like total shares outstanding, free float shares, industry classification, market classification, etc;
|
||||
4. The high frequency data containing price and volume of each stock containing open close high low volume vwap in each minute;
|
||||
5. The Consensus Expectations Factor containing the consensus expectations of the analysts about the future performance of the company.
|
||||
|
||||
|
||||
A viable factor should satisfy the following conditions:
|
||||
1. The factor should be able to be calculated in daily frequency;
|
||||
2. The factor should be able to be calculated based on each stock;
|
||||
3. The factor should be able to be calculated based on the source data provided by user.
|
||||
|
||||
You should give decision to each factor provided by the user. You should reject the factor based on very solid reason.
|
||||
Please return true to the viable factor and false to the non-viable factor.
|
||||
|
||||
Notice, you can just return part of the factors due to token limit. Your factor name should be the same as the user's factor name.
|
||||
|
||||
Please respond with your decision in JSON format. Just respond the output json string without any interaction and explanation.
|
||||
The JSON schema should include:
|
||||
```json
|
||||
{
|
||||
"Name to factor 1":
|
||||
{
|
||||
"viability": true,
|
||||
"reason": "The reason to the viability of this factor"
|
||||
},
|
||||
"Name to factor 2":
|
||||
{
|
||||
"viability": false,
|
||||
"reason": "The reason to the non-viability of this factor"
|
||||
}
|
||||
"Name to factor 3":
|
||||
{
|
||||
"viability": true,
|
||||
"reason": "The reason to the viability of this factor"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
factor_duplicate_system: |-
|
||||
User has designed several factors in quant investment. Please help the user to duplicate these factors.
|
||||
These factors are used to build a daily frequency strategy in China A-share market.
|
||||
|
||||
User will provide a pandas dataframe like table containing following information:
|
||||
1. The name of the factor;
|
||||
2. The simple description of the factor;
|
||||
3. The formulation of the factor in latex format;
|
||||
4. The description to the variables and functions in the formulation of the factor.
|
||||
|
||||
User wants to find whether there are duplicated groups. The factors in a duplicate group should satisfy the following conditions:
|
||||
1. They might differ in the name, description, formulation, or the description to the variables and functions in the formulation, some upper or lower case difference is included;
|
||||
2. They should be talking about exactly the same factor;
|
||||
3. If horizon information like 1 day, 5 days, 10 days, etc is provided, the horizon information should be the same.
|
||||
|
||||
To make your response valid, we have some very important constraint for you to follow! Listed here:
|
||||
1. You should be very confident to put duplicated factors into a group;
|
||||
2. A group should contain at least two factors;
|
||||
3. To a factor which has no duplication, don't put them into your response;
|
||||
4. To avoid merging too many similar factor, don't put more than ten factors into a group!
|
||||
You should always follow the above constraints to make your response valid.
|
||||
|
||||
Your response JSON schema should include:
|
||||
```json
|
||||
[
|
||||
[
|
||||
"factor name 1",
|
||||
"factor name 2"
|
||||
],
|
||||
[
|
||||
"factor name 5",
|
||||
"factor name 6"
|
||||
],
|
||||
[
|
||||
"factor name 7",
|
||||
"factor name 8",
|
||||
"factor name 9"
|
||||
]
|
||||
]
|
||||
```
|
||||
Your response is a list of lists. Each list represents a duplicate group containing all the factor names in this group.
|
||||
The factor names in the list should be unique and the factor names should be the same as the user's factor name.
|
||||
To avoid reaching token limit, don't respond more than fifty groups in one response. You should respond the output json string without any interaction and explanation.
|
||||
Reference in New Issue
Block a user