From c17244a317644adc7bbcfac4e15a06b942034632 Mon Sep 17 00:00:00 2001 From: WinstonLiyt <104308117+WinstonLiyt@users.noreply.github.com> Date: Sat, 20 Jul 2024 12:31:40 +0800 Subject: [PATCH] Extract factors from financial reports loop finished. (#90) - Extract factors from financial reports loop finished. - Fix some small bugs. --- rdagent/app/qlib_rd_loop/RDAgent.py | 2 +- rdagent/app/qlib_rd_loop/conf.py | 5 + .../app/qlib_rd_loop/factor_from_report.py | 121 +++++++++++++++ .../app/qlib_rd_loop/factor_from_report_sh.py | 144 ++++++++++++++++++ rdagent/app/qlib_rd_loop/prompts.yaml | 15 ++ .../coder/factor_coder/CoSTEER/evaluators.py | 10 +- .../factor_coder/CoSTEER/evolving_strategy.py | 6 +- .../scenarios/qlib/developer/factor_runner.py | 4 +- rdagent/scenarios/qlib/developer/feedback.py | 46 +++++- rdagent/scenarios/qlib/prompts.yaml | 23 ++- 10 files changed, 357 insertions(+), 19 deletions(-) create mode 100644 rdagent/app/qlib_rd_loop/factor_from_report.py create mode 100644 rdagent/app/qlib_rd_loop/factor_from_report_sh.py create mode 100644 rdagent/app/qlib_rd_loop/prompts.yaml diff --git a/rdagent/app/qlib_rd_loop/RDAgent.py b/rdagent/app/qlib_rd_loop/RDAgent.py index d125b429..661448b9 100644 --- a/rdagent/app/qlib_rd_loop/RDAgent.py +++ b/rdagent/app/qlib_rd_loop/RDAgent.py @@ -44,7 +44,7 @@ class Model_RD_Agent: return exp def generate_feedback(self, exp, hypothesis): - feedback = self.qlib_model_summarizer.generateFeedback(exp, hypothesis, self.trace) + feedback = self.qlib_model_summarizer.generate_feedback(exp, hypothesis, self.trace) self.dump_objects(exp=exp, hypothesis=hypothesis, feedback=feedback, trace=self.trace, filename='step_feedback.pkl') return feedback diff --git a/rdagent/app/qlib_rd_loop/conf.py b/rdagent/app/qlib_rd_loop/conf.py index df9d1dd5..60d63dbb 100644 --- a/rdagent/app/qlib_rd_loop/conf.py +++ b/rdagent/app/qlib_rd_loop/conf.py @@ -30,5 +30,10 @@ class PropSetting(BaseSettings): py_bin: str = "/usr/bin/python" local_qlib_folder: Path = Path("/home/rdagent/qlib") + origin_report_path: str = "data/report_origin" + local_report_path: str = "data/report" + report_result_json_file_path: str = "git_ignore_folder/res_dict.json" + progress_file_path: str = "git_ignore_folder/progress.pkl" + PROP_SETTING = PropSetting() diff --git a/rdagent/app/qlib_rd_loop/factor_from_report.py b/rdagent/app/qlib_rd_loop/factor_from_report.py new file mode 100644 index 00000000..4ea9410a --- /dev/null +++ b/rdagent/app/qlib_rd_loop/factor_from_report.py @@ -0,0 +1,121 @@ +import json +from pathlib import Path +import pickle +from dotenv import load_dotenv +from jinja2 import Environment, StrictUndefined +import pandas as pd + +from rdagent.app.qlib_rd_loop.conf import PROP_SETTING +from rdagent.components.document_reader.document_reader import load_and_process_pdfs_by_langchain +from rdagent.core.prompts import Prompts +from rdagent.core.scenario import Scenario +from rdagent.core.utils import import_class +from rdagent.log import rdagent_logger as logger +from rdagent.oai.llm_utils import APIBackend +from rdagent.scenarios.qlib.developer.factor_coder import QlibFactorCoSTEER +from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario, QlibFactorExperiment +from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import ( + FactorExperimentLoaderFromPDFfiles, + classify_report_from_dict, +) + +from rdagent.core.proposal import ( + Hypothesis2Experiment, + HypothesisExperiment2Feedback, + HypothesisGen, + Hypothesis, + Trace, +) + +from rdagent.core.developer import Developer + +assert load_dotenv() + +scen: Scenario = import_class(PROP_SETTING.factor_scen)() + +hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.factor_hypothesis_gen)(scen) + +hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.factor_hypothesis2experiment)() + +qlib_factor_coder: Developer = import_class(PROP_SETTING.factor_coder)(scen) + +qlib_factor_runner: Developer = import_class(PROP_SETTING.factor_runner)(scen) + +qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen) + +with open(PROP_SETTING.report_result_json_file_path, 'r') as f: + judge_pdf_data = json.load(f) + +prompts_path = Path(__file__).parent / "prompts.yaml" +prompts = Prompts(file_path=prompts_path) + +def generate_hypothesis(factor_result: dict, report_content: str) -> str: + system_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render() + user_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["user"]).render( + factor_descriptions=json.dumps(factor_result), + report_content=report_content + ) + + response = APIBackend().build_messages_and_create_chat_completion( + user_prompt=user_prompt, + system_prompt=system_prompt, + json_mode=True, + ) + + response_json = json.loads(response) + hypothesis_text = response_json.get("hypothesis", "No hypothesis generated.") + reason_text = response_json.get("reason", "No reason provided.") + + return Hypothesis(hypothesis=hypothesis_text, reason=reason_text) + +def extract_factors_and_implement(report_file_path: str) -> tuple: + scenario = QlibFactorScenario() + + with logger.tag("extract_factors_and_implement"): + with logger.tag("load_factor_tasks"): + + exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path) + if exp is None or exp.sub_tasks == []: + return None, None + + docs_dict = load_and_process_pdfs_by_langchain(Path(report_file_path)) + + factor_result = { + task.factor_name: { + "description": task.factor_description, + "formulation": task.factor_formulation, + "variables": task.variables, + "resources": task.factor_resources + } + for task in exp.sub_tasks + } + + report_content = "\n".join(docs_dict.values()) + hypothesis = generate_hypothesis(factor_result, report_content) + + return exp, hypothesis + +trace = Trace(scen=scen) + +for file_path, attributes in judge_pdf_data.items(): + if attributes["class"] == 1: + report_file_path = Path(file_path.replace(PROP_SETTING.origin_report_path, PROP_SETTING.local_report_path)) + if report_file_path.exists(): + logger.info(f"Processing {report_file_path}") + exp, hypothesis = extract_factors_and_implement(str(report_file_path)) + if exp is None: + continue + exp.based_experiments = [t[1] for t in trace.hist if t[2]] + if len(exp.based_experiments) == 0: + exp.based_experiments.append(QlibFactorExperiment(sub_tasks=[])) + exp = qlib_factor_coder.develop(exp) + exp = qlib_factor_runner.develop(exp) + if exp is None: + logger.error(f"Factor extraction failed for {report_file_path}. Skipping to the next report.") + continue + feedback = qlib_factor_summarizer.generate_feedback(exp, hypothesis, trace) + + trace.hist.append((hypothesis, exp, feedback)) + logger.info(f"Processed {report_file_path}: Result: {exp}") + else: + logger.error(f"File not found: {report_file_path}") diff --git a/rdagent/app/qlib_rd_loop/factor_from_report_sh.py b/rdagent/app/qlib_rd_loop/factor_from_report_sh.py new file mode 100644 index 00000000..504b16be --- /dev/null +++ b/rdagent/app/qlib_rd_loop/factor_from_report_sh.py @@ -0,0 +1,144 @@ +# TODO: we should have more advanced mechanism to handle such requirements for saving sessions. +import json +from pathlib import Path +import pickle +from dotenv import load_dotenv +from jinja2 import Environment, StrictUndefined +import pandas as pd + +from rdagent.app.qlib_rd_loop.conf import PROP_SETTING +from rdagent.components.document_reader.document_reader import load_and_process_pdfs_by_langchain +from rdagent.core.prompts import Prompts +from rdagent.core.scenario import Scenario +from rdagent.core.utils import import_class +from rdagent.log import rdagent_logger as logger +from rdagent.oai.llm_utils import APIBackend +from rdagent.scenarios.qlib.developer.factor_coder import QlibFactorCoSTEER +from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorScenario, QlibFactorExperiment +from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import ( + FactorExperimentLoaderFromPDFfiles, + classify_report_from_dict, +) + +from rdagent.core.proposal import ( + Hypothesis2Experiment, + HypothesisExperiment2Feedback, + HypothesisGen, + Hypothesis, + Trace, +) + +from rdagent.core.developer import Developer + +assert load_dotenv() + +scen: Scenario = import_class(PROP_SETTING.factor_scen)() + +hypothesis_gen: HypothesisGen = import_class(PROP_SETTING.factor_hypothesis_gen)(scen) + +hypothesis2experiment: Hypothesis2Experiment = import_class(PROP_SETTING.factor_hypothesis2experiment)() + +qlib_factor_coder: Developer = import_class(PROP_SETTING.factor_coder)(scen) + +qlib_factor_runner: Developer = import_class(PROP_SETTING.factor_runner)(scen) + +qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.factor_summarizer)(scen) + +with open(PROP_SETTING.report_result_json_file_path, 'r') as f: + judge_pdf_data = json.load(f) + +prompts_path = Path(__file__).parent / "prompts.yaml" +prompts = Prompts(file_path=prompts_path) + +def save_progress(trace, current_index): + with open(PROP_SETTING.progress_file, "wb") as f: + pickle.dump((trace, current_index), f) + +def load_progress(): + if Path(PROP_SETTING.progress_file).exists(): + with open(PROP_SETTING.progress_file, "rb") as f: + return pickle.load(f) + return Trace(scen=scen), 0 + +def generate_hypothesis(factor_result: dict, report_content: str) -> str: + system_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render() + user_prompt = Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["user"]).render( + factor_descriptions=json.dumps(factor_result), + report_content=report_content + ) + + response = APIBackend().build_messages_and_create_chat_completion( + user_prompt=user_prompt, + system_prompt=system_prompt, + json_mode=True, + ) + + response_json = json.loads(response) + hypothesis_text = response_json.get("hypothesis", "No hypothesis generated.") + reason_text = response_json.get("reason", "No reason provided.") + + return Hypothesis(hypothesis=hypothesis_text, reason=reason_text) + +def extract_factors_and_implement(report_file_path: str) -> tuple: + scenario = QlibFactorScenario() + + with logger.tag("extract_factors_and_implement"): + with logger.tag("load_factor_tasks"): + + exp = FactorExperimentLoaderFromPDFfiles().load(report_file_path) + if exp is None or exp.sub_tasks == []: + return None, None + + docs_dict = load_and_process_pdfs_by_langchain(Path(report_file_path)) + + factor_result = { + task.factor_name: { + "description": task.factor_description, + "formulation": task.factor_formulation, + "variables": task.variables, + "resources": task.factor_resources + } + for task in exp.sub_tasks + } + + report_content = "\n".join(docs_dict.values()) + hypothesis = generate_hypothesis(factor_result, report_content) + + return exp, hypothesis + +trace, start_index = load_progress() + +try: + judge_pdf_data_items = list(judge_pdf_data.items()) + for index in range(start_index, len(judge_pdf_data_items)): + if index > 1000: + break + file_path, attributes = judge_pdf_data_items[index] + if attributes["class"] == 1: + report_file_path = Path(file_path.replace(PROP_SETTING.origin_report_path, PROP_SETTING.local_report_path)) + if report_file_path.exists(): + logger.info(f"Processing {report_file_path}") + exp, hypothesis = extract_factors_and_implement(str(report_file_path)) + if exp is None: + continue + exp.based_experiments = [t[1] for t in trace.hist if t[2]] + if len(exp.based_experiments) == 0: + exp.based_experiments.append(QlibFactorExperiment(sub_tasks=[])) + exp = qlib_factor_coder.develop(exp) + exp = qlib_factor_runner.develop(exp) + if exp is None: + logger.error(f"Factor extraction failed for {report_file_path}. Skipping to the next report.") + continue + feedback = qlib_factor_summarizer.generate_feedback(exp, hypothesis, trace) + + trace.hist.append((hypothesis, exp, feedback)) + logger.info(f"Processed {report_file_path}: Result: {exp}") + + # Save progress after processing each report + save_progress(trace, index + 1) + else: + logger.error(f"File not found: {report_file_path}") +except Exception as e: + logger.error(f"An error occurred: {e}") + save_progress(trace, index) + raise diff --git a/rdagent/app/qlib_rd_loop/prompts.yaml b/rdagent/app/qlib_rd_loop/prompts.yaml new file mode 100644 index 00000000..1909087c --- /dev/null +++ b/rdagent/app/qlib_rd_loop/prompts.yaml @@ -0,0 +1,15 @@ +hypothesis_generation: + system: |- + You are an expert in financial analysis. Your task is to generate a well-reasoned hypothesis based on the provided financial factors and report content. + Please ensure your response is in JSON format as shown below: + { + "hypothesis": "A clear and concise hypothesis based on the provided information.", + "reason": "A detailed explanation supporting the generated hypothesis." + } + + user: |- + The following are the financial factors and their descriptions: + {{ factor_descriptions }} + + The report content is as follows: + {{ report_content }} \ No newline at end of file diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py index b38348fb..00b59d8e 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evaluators.py @@ -334,6 +334,13 @@ class FactorValueEvaluator(FactorEvaluator): ) -> Tuple: conclusions = [] + # Initialize result variables + single_column_result = None + same_index_result = None + output_format_result = None + equal_value_ratio_result = 0 + high_correlation_result = False + # Check if both dataframe has only one columns feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation) conclusions.append(feedback_str) @@ -373,9 +380,6 @@ class FactorValueEvaluator(FactorEvaluator): high_correlation_result = False feedback_str = "The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless" conclusions.append(feedback_str) - else: - equal_value_ratio_result = 0 - high_correlation_result = False # Combine all conclusions into a single string conclusion_str = "\n".join(conclusions) diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py b/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py index 97c225ff..1493cfac 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py @@ -68,9 +68,9 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): # if the number of factors to be implemented is larger than the limit, we need to select some of them if FACTOR_IMPLEMENT_SETTINGS.select_ratio < 1: # if the number of loops is equal to the select_loop, we need to select some of them - implementation_factors_per_round = int( - FACTOR_IMPLEMENT_SETTINGS.select_ratio * len(to_be_finished_task_index) - ) + implementation_factors_per_round = round(FACTOR_IMPLEMENT_SETTINGS.select_ratio * len(to_be_finished_task_index) + 0.5) # ceilling + implementation_factors_per_round = min(implementation_factors_per_round, len(to_be_finished_task_index)) # but not exceed the total number of tasks + if FACTOR_IMPLEMENT_SETTINGS.select_method == "random": to_be_finished_task_index = RandomSelect( to_be_finished_task_index, diff --git a/rdagent/scenarios/qlib/developer/factor_runner.py b/rdagent/scenarios/qlib/developer/factor_runner.py index 8fa77a44..d08394ec 100644 --- a/rdagent/scenarios/qlib/developer/factor_runner.py +++ b/rdagent/scenarios/qlib/developer/factor_runner.py @@ -60,7 +60,9 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): new_factors = self.process_factor_data(exp) if new_factors.empty: - raise FactorEmptyError("No valid factor data found to merge.") + # raise FactorEmptyException("No valid factor data found to merge.") + logger.error("No valid factor data found to merge.") + return None # Combine the SOTA factor and new factors if SOTA factor exists if SOTA_factor is not None and not SOTA_factor.empty: diff --git a/rdagent/scenarios/qlib/developer/feedback.py b/rdagent/scenarios/qlib/developer/feedback.py index 6919b075..612e09f1 100644 --- a/rdagent/scenarios/qlib/developer/feedback.py +++ b/rdagent/scenarios/qlib/developer/feedback.py @@ -1,10 +1,8 @@ -# TODO: -# Implement to feedback. - import json from pathlib import Path from jinja2 import Environment, StrictUndefined +import pandas as pd from rdagent.core.experiment import Experiment from rdagent.core.prompts import Prompts @@ -21,6 +19,39 @@ from rdagent.utils import convert2bool feedback_prompts = Prompts(file_path=Path(__file__).parent.parent / "prompts.yaml") DIRNAME = Path(__file__).absolute().resolve().parent +def process_results(current_result, sota_result): + # Convert the results to dataframes + current_df = pd.DataFrame(current_result) + sota_df = pd.DataFrame(sota_result) + + # Set the metric as the index + current_df.index.name = 'metric' + sota_df.index.name = 'metric' + + # Rename the value column to reflect the result type + current_df.rename(columns={'0': 'Current Result'}, inplace=True) + sota_df.rename(columns={'0': 'SOTA Result'}, inplace=True) + + # Combine the dataframes on the Metric index + combined_df = pd.concat([current_df, sota_df], axis=1) + + # Select important metrics for comparison + important_metrics = [ + 'Rank ICIR', + '1day.excess_return_without_cost.max_drawdown', + '1day.excess_return_without_cost.information_ratio', + '1day.excess_return_without_cost.annualized_return', + '1day.excess_return_with_cost.max_drawdown', + '1day.excess_return_with_cost.information_ratio', + '1day.excess_return_with_cost.annualized_return', + 'IC', + ] + + # Filter the combined DataFrame to retain only the important metrics + filtered_combined_df = combined_df.loc[important_metrics] + + return filtered_combined_df.to_dict() + class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): def generate_feedback(self, exp: Experiment, hypothesis: Hypothesis, trace: Trace) -> HypothesisFeedback: @@ -41,6 +72,10 @@ class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): tasks_factors = [task.get_task_information() for task in exp.sub_tasks] sota_result = exp.based_experiments[-1].result + # Process the results to filter important metrics + combined_result = process_results(current_result, sota_result) + logger.info(f"combined_result: {combined_result}") + # Generate the system prompt sys_prompt = ( Environment(undefined=StrictUndefined) @@ -55,8 +90,9 @@ class QlibFactorHypothesisExperiment2Feedback(HypothesisExperiment2Feedback): .render( hypothesis_text=hypothesis_text, task_details=tasks_factors, - current_result=current_result, - sota_result=sota_result, + # current_result=current_result, + # sota_result=sota_result, + combined_result=combined_result, ) ) diff --git a/rdagent/scenarios/qlib/prompts.yaml b/rdagent/scenarios/qlib/prompts.yaml index 8ac4fba6..292bbf07 100644 --- a/rdagent/scenarios/qlib/prompts.yaml +++ b/rdagent/scenarios/qlib/prompts.yaml @@ -108,7 +108,7 @@ factor_feedback_generation: The task is described in the following scenario: {{ scenario }} You will receive a hypothesis, multiple tasks with their factors, and some results. - Your feedback should specify whether the current result supports or refutes the hypothesis, compare it with previous results, and suggest improvements or new directions. + Your feedback should specify whether the current result supports or refutes the hypothesis, compare it with previous SOTA results, and suggest improvements or new directions. Please provide detailed and constructive feedback for the future exploration. Please respond in JSON format, and example JSON Structure for Result Analysis: { @@ -123,15 +123,26 @@ factor_feedback_generation: {{ hypothesis_text }} Tasks and Factors: {{ task_details }} - Current Result: - {{ current_result }} - SOTA Result: - {{ sota_result }} - Analyze the current result in the context of its ability to: + Combined Results: + {{ combined_result }} + Analyze the combined result in the context of its ability to: 1. Support or refute the hypothesis. 2. Show improvement or deterioration compared to the last experiment. 3. Demonstrate positive or negative effects when compared to Alpha158. + Evaluation Metrics Explanations: + Below are the financial meanings of each metric, which should be used to judge the results: + + - Rank ICIR: Evaluates the stability and average level of Rank IC. Rank ICIR = mean(Rank IC) / std(Rank IC). + - 1day.excess_return_without_cost.max_drawdown: Measures the maximum loss from a peak to a trough without considering transaction costs. + - 1day.excess_return_without_cost.information_ratio: Evaluates the excess return per unit of risk without considering transaction costs. + - 1day.excess_return_with_cost.max_drawdown: Measures the maximum loss from a peak to a trough considering transaction costs. + - 1day.excess_return_without_cost.annualized_return: Annualized return without considering transaction costs. + - 1day.excess_return_with_cost.annualized_return: Annualized return considering transaction costs. + - IC: Measures the correlation between predicted returns (\hat{y}) and actual returns (y), using Pearson correlation. + - 1day.excess_return_with_cost.information_ratio: Evaluates the excess return per unit of risk considering transaction costs. + + When judging the results, prioritize metrics that consider transaction costs (with cost), as they provide a more accurate representation of real-world performance. Among these, the annualized return considering transaction costs is particularly important as it gives a clear picture of long-term profitability. Provide detailed feedback and recommend whether to replace the best result if the new factor proves superior. model_feedback_generation: