mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-07-27 15:37:44 +00:00
Extract factors from financial reports loop finished. (#90)
- Extract factors from financial reports loop finished. - Fix some small bugs.
This commit is contained in:
WinstonLiyt
@@ -44,7 +44,7 @@ class Model_RD_Agent:
|
||||
return exp
|
||||
|
||||
def generate_feedback(self, exp, hypothesis):
|
||||
feedback = self.qlib_model_summarizer.generateFeedback(exp, hypothesis, self.trace)
|
||||
feedback = self.qlib_model_summarizer.generate_feedback(exp, hypothesis, self.trace)
|
||||
self.dump_objects(exp=exp, hypothesis=hypothesis, feedback=feedback, trace=self.trace, filename='step_feedback.pkl')
|
||||
return feedback
|
||||
|
||||
|
||||
@@ -30,5 +30,10 @@ class PropSetting(BaseSettings):
|
||||
py_bin: str = "/usr/bin/python"
|
||||
local_qlib_folder: Path = Path("/home/rdagent/qlib")
|
||||
|
||||
origin_report_path: str = "data/report_origin"
|
||||
local_report_path: str = "data/report"
|
||||
report_result_json_file_path: str = "git_ignore_folder/res_dict.json"
|
||||
progress_file_path: str = "git_ignore_folder/progress.pkl"
|
||||
|
||||
|
||||
PROP_SETTING = PropSetting()
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
import pickle
|
||||
from dotenv import load_dotenv
|
||||
from jinja2 import Environment, StrictUndefined
|
||||
import pandas as pd
|
||||
|
||||
from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
|
||||
from rdagent.components.document_reader.document_reader import load_and_process_pdfs_by_langchain
|
||||
from rdagent.core.prompts import Prompts
|
||||
from rdagent.core.scenario import Scenario
|
||||
from rdagent.core.utils import import_class
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
from rdagent.scenarios.qlib.developer.factor_coder import QlibFactorCoSTEER
|
||||
from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario, QlibFactorExperiment
|
||||
from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import (
|
||||
FactorExperimentLoaderFromPDFfiles,
|
||||
classify_report_from_dict,
|
||||
)
|
||||
|
||||
from rdagent.core.proposal import (
|
||||
Hypothesis2Experiment,
|
||||
HypothesisExperiment2Feedback,
|
||||
HypothesisGen,
|
||||
Hypothesis,
|
||||
Trace,
|
||||
)
|
||||
|
||||
from rdagent.core.developer import Developer
|
||||
|
||||
assert load_dotenv()
|
||||
|
||||
scen: Scenario = import_class(PROP_SETTING.factor_scen)()
|
||||
|
||||
hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.factor_hypothesis_gen)(scen)
|
||||
|
||||
hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.factor_hypothesis2experiment)()
|
||||
|
||||
qlib_factor_coder: Developer = import_class(PROP_SETTING.factor_coder)(scen)
|
||||
|
||||
qlib_factor_runner: Developer = import_class(PROP_SETTING.factor_runner)(scen)
|
||||
|
||||
qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen)
|
||||
|
||||
with open(PROP_SETTING.report_result_json_file_path, 'r') as f:
|
||||
judge_pdf_data = json.load(f)
|
||||
|
||||
prompts_path = Path(__file__).parent / "prompts.yaml"
|
||||
prompts = Prompts(file_path=prompts_path)
|
||||
|
||||
def generate_hypothesis(factor_result: dict, report_content: str) -> str:
|
||||
system_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render()
|
||||
user_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["user"]).render(
|
||||
factor_descriptions=json.dumps(factor_result),
|
||||
report_content=report_content
|
||||
)
|
||||
|
||||
response = APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
json_mode=True,
|
||||
)
|
||||
|
||||
response_json = json.loads(response)
|
||||
hypothesis_text = response_json.get("hypothesis", "No hypothesis generated.")
|
||||
reason_text = response_json.get("reason", "No reason provided.")
|
||||
|
||||
return Hypothesis(hypothesis=hypothesis_text, reason=reason_text)
|
||||
|
||||
def extract_factors_and_implement(report_file_path: str) -> tuple:
|
||||
scenario = QlibFactorScenario()
|
||||
|
||||
with logger.tag("extract_factors_and_implement"):
|
||||
with logger.tag("load_factor_tasks"):
|
||||
|
||||
exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path)
|
||||
if exp is None or exp.sub_tasks == []:
|
||||
return None, None
|
||||
|
||||
docs_dict = load_and_process_pdfs_by_langchain(Path(report_file_path))
|
||||
|
||||
factor_result = {
|
||||
task.factor_name: {
|
||||
"description": task.factor_description,
|
||||
"formulation": task.factor_formulation,
|
||||
"variables": task.variables,
|
||||
"resources": task.factor_resources
|
||||
}
|
||||
for task in exp.sub_tasks
|
||||
}
|
||||
|
||||
report_content = "\n".join(docs_dict.values())
|
||||
hypothesis = generate_hypothesis(factor_result, report_content)
|
||||
|
||||
return exp, hypothesis
|
||||
|
||||
trace = Trace(scen=scen)
|
||||
|
||||
for file_path, attributes in judge_pdf_data.items():
|
||||
if attributes["class"] == 1:
|
||||
report_file_path = Path(file_path.replace(PROP_SETTING.origin_report_path, PROP_SETTING.local_report_path))
|
||||
if report_file_path.exists():
|
||||
logger.info(f"Processing {report_file_path}")
|
||||
exp, hypothesis = extract_factors_and_implement(str(report_file_path))
|
||||
if exp is None:
|
||||
continue
|
||||
exp.based_experiments = [t[1] for t in trace.hist if t[2]]
|
||||
if len(exp.based_experiments) == 0:
|
||||
exp.based_experiments.append(QlibFactorExperiment(sub_tasks=[]))
|
||||
exp = qlib_factor_coder.develop(exp)
|
||||
exp = qlib_factor_runner.develop(exp)
|
||||
if exp is None:
|
||||
logger.error(f"Factor extraction failed for {report_file_path}. Skipping to the next report.")
|
||||
continue
|
||||
feedback = qlib_factor_summarizer.generate_feedback(exp, hypothesis, trace)
|
||||
|
||||
trace.hist.append((hypothesis, exp, feedback))
|
||||
logger.info(f"Processed {report_file_path}: Result: {exp}")
|
||||
else:
|
||||
logger.error(f"File not found: {report_file_path}")
|
||||
@@ -0,0 +1,144 @@
|
||||
# TODO: we should have more advanced mechanism to handle such requirements for saving sessions.
|
||||
import json
|
||||
from pathlib import Path
|
||||
import pickle
|
||||
from dotenv import load_dotenv
|
||||
from jinja2 import Environment, StrictUndefined
|
||||
import pandas as pd
|
||||
|
||||
from rdagent.app.qlib_rd_loop.conf import PROP_SETTING
|
||||
from rdagent.components.document_reader.document_reader import load_and_process_pdfs_by_langchain
|
||||
from rdagent.core.prompts import Prompts
|
||||
from rdagent.core.scenario import Scenario
|
||||
from rdagent.core.utils import import_class
|
||||
from rdagent.log import rdagent_logger as logger
|
||||
from rdagent.oai.llm_utils import APIBackend
|
||||
from rdagent.scenarios.qlib.developer.factor_coder import QlibFactorCoSTEER
|
||||
from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario, QlibFactorExperiment
|
||||
from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import (
|
||||
FactorExperimentLoaderFromPDFfiles,
|
||||
classify_report_from_dict,
|
||||
)
|
||||
|
||||
from rdagent.core.proposal import (
|
||||
Hypothesis2Experiment,
|
||||
HypothesisExperiment2Feedback,
|
||||
HypothesisGen,
|
||||
Hypothesis,
|
||||
Trace,
|
||||
)
|
||||
|
||||
from rdagent.core.developer import Developer
|
||||
|
||||
assert load_dotenv()
|
||||
|
||||
scen: Scenario = import_class(PROP_SETTING.factor_scen)()
|
||||
|
||||
hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.factor_hypothesis_gen)(scen)
|
||||
|
||||
hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.factor_hypothesis2experiment)()
|
||||
|
||||
qlib_factor_coder: Developer = import_class(PROP_SETTING.factor_coder)(scen)
|
||||
|
||||
qlib_factor_runner: Developer = import_class(PROP_SETTING.factor_runner)(scen)
|
||||
|
||||
qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen)
|
||||
|
||||
with open(PROP_SETTING.report_result_json_file_path, 'r') as f:
|
||||
judge_pdf_data = json.load(f)
|
||||
|
||||
prompts_path = Path(__file__).parent / "prompts.yaml"
|
||||
prompts = Prompts(file_path=prompts_path)
|
||||
|
||||
def save_progress(trace, current_index):
|
||||
with open(PROP_SETTING.progress_file, "wb") as f:
|
||||
pickle.dump((trace, current_index), f)
|
||||
|
||||
def load_progress():
|
||||
if Path(PROP_SETTING.progress_file).exists():
|
||||
with open(PROP_SETTING.progress_file, "rb") as f:
|
||||
return pickle.load(f)
|
||||
return Trace(scen=scen), 0
|
||||
|
||||
def generate_hypothesis(factor_result: dict, report_content: str) -> str:
|
||||
system_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render()
|
||||
user_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["user"]).render(
|
||||
factor_descriptions=json.dumps(factor_result),
|
||||
report_content=report_content
|
||||
)
|
||||
|
||||
response = APIBackend().build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
json_mode=True,
|
||||
)
|
||||
|
||||
response_json = json.loads(response)
|
||||
hypothesis_text = response_json.get("hypothesis", "No hypothesis generated.")
|
||||
reason_text = response_json.get("reason", "No reason provided.")
|
||||
|
||||
return Hypothesis(hypothesis=hypothesis_text, reason=reason_text)
|
||||
|
||||
def extract_factors_and_implement(report_file_path: str) -> tuple:
|
||||
scenario = QlibFactorScenario()
|
||||
|
||||
with logger.tag("extract_factors_and_implement"):
|
||||
with logger.tag("load_factor_tasks"):
|
||||
|
||||
exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path)
|
||||
if exp is None or exp.sub_tasks == []:
|
||||
return None, None
|
||||
|
||||
docs_dict = load_and_process_pdfs_by_langchain(Path(report_file_path))
|
||||
|
||||
factor_result = {
|
||||
task.factor_name: {
|
||||
"description": task.factor_description,
|
||||
"formulation": task.factor_formulation,
|
||||
"variables": task.variables,
|
||||
"resources": task.factor_resources
|
||||
}
|
||||
for task in exp.sub_tasks
|
||||
}
|
||||
|
||||
report_content = "\n".join(docs_dict.values())
|
||||
hypothesis = generate_hypothesis(factor_result, report_content)
|
||||
|
||||
return exp, hypothesis
|
||||
|
||||
trace, start_index = load_progress()
|
||||
|
||||
try:
|
||||
judge_pdf_data_items = list(judge_pdf_data.items())
|
||||
for index in range(start_index, len(judge_pdf_data_items)):
|
||||
if index > 1000:
|
||||
break
|
||||
file_path, attributes = judge_pdf_data_items[index]
|
||||
if attributes["class"] == 1:
|
||||
report_file_path = Path(file_path.replace(PROP_SETTING.origin_report_path, PROP_SETTING.local_report_path))
|
||||
if report_file_path.exists():
|
||||
logger.info(f"Processing {report_file_path}")
|
||||
exp, hypothesis = extract_factors_and_implement(str(report_file_path))
|
||||
if exp is None:
|
||||
continue
|
||||
exp.based_experiments = [t[1] for t in trace.hist if t[2]]
|
||||
if len(exp.based_experiments) == 0:
|
||||
exp.based_experiments.append(QlibFactorExperiment(sub_tasks=[]))
|
||||
exp = qlib_factor_coder.develop(exp)
|
||||
exp = qlib_factor_runner.develop(exp)
|
||||
if exp is None:
|
||||
logger.error(f"Factor extraction failed for {report_file_path}. Skipping to the next report.")
|
||||
continue
|
||||
feedback = qlib_factor_summarizer.generate_feedback(exp, hypothesis, trace)
|
||||
|
||||
trace.hist.append((hypothesis, exp, feedback))
|
||||
logger.info(f"Processed {report_file_path}: Result: {exp}")
|
||||
|
||||
# Save progress after processing each report
|
||||
save_progress(trace, index + 1)
|
||||
else:
|
||||
logger.error(f"File not found: {report_file_path}")
|
||||
except Exception as e:
|
||||
logger.error(f"An error occurred: {e}")
|
||||
save_progress(trace, index)
|
||||
raise
|
||||
@@ -0,0 +1,15 @@
|
||||
hypothesis_generation:
|
||||
system: |-
|
||||
You are an expert in financial analysis. Your task is to generate a well-reasoned hypothesis based on the provided financial factors and report content.
|
||||
Please ensure your response is in JSON format as shown below:
|
||||
{
|
||||
"hypothesis": "A clear and concise hypothesis based on the provided information.",
|
||||
"reason": "A detailed explanation supporting the generated hypothesis."
|
||||
}
|
||||
|
||||
user: |-
|
||||
The following are the financial factors and their descriptions:
|
||||
{{ factor_descriptions }}
|
||||
|
||||
The report content is as follows:
|
||||
{{ report_content }}
|
||||
@@ -334,6 +334,13 @@ class FactorValueEvaluator(FactorEvaluator):
|
||||
) -> Tuple:
|
||||
conclusions = []
|
||||
|
||||
# Initialize result variables
|
||||
single_column_result = None
|
||||
same_index_result = None
|
||||
output_format_result = None
|
||||
equal_value_ratio_result = 0
|
||||
high_correlation_result = False
|
||||
|
||||
# Check if both dataframe has only one columns
|
||||
feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation)
|
||||
conclusions.append(feedback_str)
|
||||
@@ -373,9 +380,6 @@ class FactorValueEvaluator(FactorEvaluator):
|
||||
high_correlation_result = False
|
||||
feedback_str = "The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless"
|
||||
conclusions.append(feedback_str)
|
||||
else:
|
||||
equal_value_ratio_result = 0
|
||||
high_correlation_result = False
|
||||
|
||||
# Combine all conclusions into a single string
|
||||
conclusion_str = "\n".join(conclusions)
|
||||
|
||||
@@ -68,9 +68,9 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
|
||||
# if the number of factors to be implemented is larger than the limit, we need to select some of them
|
||||
if FACTOR_IMPLEMENT_SETTINGS.select_ratio < 1:
|
||||
# if the number of loops is equal to the select_loop, we need to select some of them
|
||||
implementation_factors_per_round = int(
|
||||
FACTOR_IMPLEMENT_SETTINGS.select_ratio * len(to_be_finished_task_index)
|
||||
)
|
||||
implementation_factors_per_round = round(FACTOR_IMPLEMENT_SETTINGS.select_ratio * len(to_be_finished_task_index) + 0.5) # ceilling
|
||||
implementation_factors_per_round = min(implementation_factors_per_round, len(to_be_finished_task_index)) # but not exceed the total number of tasks
|
||||
|
||||
if FACTOR_IMPLEMENT_SETTINGS.select_method == "random":
|
||||
to_be_finished_task_index = RandomSelect(
|
||||
to_be_finished_task_index,
|
||||
|
||||
@@ -60,7 +60,9 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]):
|
||||
new_factors = self.process_factor_data(exp)
|
||||
|
||||
if new_factors.empty:
|
||||
raise FactorEmptyError("No valid factor data found to merge.")
|
||||
# raise FactorEmptyException("No valid factor data found to merge.")
|
||||
logger.error("No valid factor data found to merge.")
|
||||
return None
|
||||
|
||||
# Combine the SOTA factor and new factors if SOTA factor exists
|
||||
if SOTA_factor is not None and not SOTA_factor.empty:
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
# TODO:
|
||||
# Implement to feedback.
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from jinja2 import Environment, StrictUndefined
|
||||
import pandas as pd
|
||||
|
||||
from rdagent.core.experiment import Experiment
|
||||
from rdagent.core.prompts import Prompts
|
||||
@@ -21,6 +19,39 @@ from rdagent.utils import convert2bool
|
||||
feedback_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml")
|
||||
DIRNAME = Path(__file__).absolute().resolve().parent
|
||||
|
||||
def process_results(current_result, sota_result):
|
||||
# Convert the results to dataframes
|
||||
current_df = pd.DataFrame(current_result)
|
||||
sota_df = pd.DataFrame(sota_result)
|
||||
|
||||
# Set the metric as the index
|
||||
current_df.index.name = 'metric'
|
||||
sota_df.index.name = 'metric'
|
||||
|
||||
# Rename the value column to reflect the result type
|
||||
current_df.rename(columns={'0': 'Current Result'}, inplace=True)
|
||||
sota_df.rename(columns={'0': 'SOTA Result'}, inplace=True)
|
||||
|
||||
# Combine the dataframes on the Metric index
|
||||
combined_df = pd.concat([current_df, sota_df], axis=1)
|
||||
|
||||
# Select important metrics for comparison
|
||||
important_metrics = [
|
||||
'Rank ICIR',
|
||||
'1day.excess_return_without_cost.max_drawdown',
|
||||
'1day.excess_return_without_cost.information_ratio',
|
||||
'1day.excess_return_without_cost.annualized_return',
|
||||
'1day.excess_return_with_cost.max_drawdown',
|
||||
'1day.excess_return_with_cost.information_ratio',
|
||||
'1day.excess_return_with_cost.annualized_return',
|
||||
'IC',
|
||||
]
|
||||
|
||||
# Filter the combined DataFrame to retain only the important metrics
|
||||
filtered_combined_df = combined_df.loc[important_metrics]
|
||||
|
||||
return filtered_combined_df.to_dict()
|
||||
|
||||
|
||||
class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
|
||||
def generate_feedback(self, exp: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback:
|
||||
@@ -41,6 +72,10 @@ class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
|
||||
tasks_factors = [task.get_task_information() for task in exp.sub_tasks]
|
||||
sota_result = exp.based_experiments[-1].result
|
||||
|
||||
# Process the results to filter important metrics
|
||||
combined_result = process_results(current_result, sota_result)
|
||||
logger.info(f"combined_result: {combined_result}")
|
||||
|
||||
# Generate the system prompt
|
||||
sys_prompt = (
|
||||
Environment(undefined=StrictUndefined)
|
||||
@@ -55,8 +90,9 @@ class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback):
|
||||
.render(
|
||||
hypothesis_text=hypothesis_text,
|
||||
task_details=tasks_factors,
|
||||
current_result=current_result,
|
||||
sota_result=sota_result,
|
||||
# current_result=current_result,
|
||||
# sota_result=sota_result,
|
||||
combined_result=combined_result,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -108,7 +108,7 @@ factor_feedback_generation:
|
||||
The task is described in the following scenario:
|
||||
{{ scenario }}
|
||||
You will receive a hypothesis, multiple tasks with their factors, and some results.
|
||||
Your feedback should specify whether the current result supports or refutes the hypothesis, compare it with previous results, and suggest improvements or new directions.
|
||||
Your feedback should specify whether the current result supports or refutes the hypothesis, compare it with previous SOTA results, and suggest improvements or new directions.
|
||||
Please provide detailed and constructive feedback for the future exploration.
|
||||
Please respond in JSON format, and example JSON Structure for Result Analysis:
|
||||
{
|
||||
@@ -123,15 +123,26 @@ factor_feedback_generation:
|
||||
{{ hypothesis_text }}
|
||||
Tasks and Factors:
|
||||
{{ task_details }}
|
||||
Current Result:
|
||||
{{ current_result }}
|
||||
SOTA Result:
|
||||
{{ sota_result }}
|
||||
Analyze the current result in the context of its ability to:
|
||||
Combined Results:
|
||||
{{ combined_result }}
|
||||
Analyze the combined result in the context of its ability to:
|
||||
1. Support or refute the hypothesis.
|
||||
2. Show improvement or deterioration compared to the last experiment.
|
||||
3. Demonstrate positive or negative effects when compared to Alpha158.
|
||||
|
||||
Evaluation Metrics Explanations:
|
||||
Below are the financial meanings of each metric, which should be used to judge the results:
|
||||
|
||||
- Rank ICIR: Evaluates the stability and average level of Rank IC. Rank ICIR = mean(Rank IC) / std(Rank IC).
|
||||
- 1day.excess_return_without_cost.max_drawdown: Measures the maximum loss from a peak to a trough without considering transaction costs.
|
||||
- 1day.excess_return_without_cost.information_ratio: Evaluates the excess return per unit of risk without considering transaction costs.
|
||||
- 1day.excess_return_with_cost.max_drawdown: Measures the maximum loss from a peak to a trough considering transaction costs.
|
||||
- 1day.excess_return_without_cost.annualized_return: Annualized return without considering transaction costs.
|
||||
- 1day.excess_return_with_cost.annualized_return: Annualized return considering transaction costs.
|
||||
- IC: Measures the correlation between predicted returns (\hat{y}) and actual returns (y), using Pearson correlation.
|
||||
- 1day.excess_return_with_cost.information_ratio: Evaluates the excess return per unit of risk considering transaction costs.
|
||||
|
||||
When judging the results, prioritize metrics that consider transaction costs (with cost), as they provide a more accurate representation of real-world performance. Among these, the annualized return considering transaction costs is particularly important as it gives a clear picture of long-term profitability.
|
||||
Provide detailed feedback and recommend whether to replace the best result if the new factor proves superior.
|
||||
|
||||
model_feedback_generation:
|
||||
|
||||
Reference in New Issue
Block a user