fix: Comprehensive update to factor extraction. (#143)

* Init todo

* update all code

* update

* Extract factors from financial reports loop finished

* Fix two small bugs.

* Delete rdagent/app/qlib_rd_loop/run_script.sh

* Minor mod

* Delete rdagent/app/qlib_rd_loop/nohup.out

* Fix a small bug in file reading.

* some updates

* Update the detailed process and prompt of factor loop.

* Evaluation & dataset

* Optimize the prompt for generating hypotheses and feedback in the factor loop.

* Generate new data

* dataset generation

* Performed further optimizations on the factor loop and report extraction loop, added log handling for both processes, and implemented a screenshot feature for report extraction.

* Update rdagent/components/coder/factor_coder/CoSTEER/evaluators.py

* Update package.txt for fitz.

* add the result

* Performed further optimizations on the factor loop and report extraction loop, added log handling for both processes, and implemented a screenshot feature for report extraction. (#100) (#102)

- Performed further optimizations on the factor loop and report extraction loop.
- Added log handling for both processes.
- Implemented a screenshot feature for report extraction.

* Analysis

* Optimized log output.

* Factor update

* A draft of the "Quick Start" section for README

* Add scenario descriptions.

* Updates

* Adjust content

* Enable logging of backtesting in Qlib and store rich-text descriptions in Trace. Support one-step debugging for factor extraction.

* Reformat analysis.py

* CI fix

* Refactor

* remove useless code

* fix bugs (#111)

* Fix two small bugs.

* Fix a merge bug.

* Fix two small bugs.

* fix some bugs.

* Fix some format bugs.

* Restore a file.

* Fix a format bug.

* draft renew of evaluators

* fix a small bug.

* fix a small bug

* Support Factor Report Loop

* Update framework for extracting factors from research reports.

* Refactor report-based factor extraction and fix minor bugs.

* fix a small bug of log.

* change some prompts

* improve factor_runner

* fix a small bug

* change some prompts

* cancel some comments

* cancel some comments and fix some bugs

---------

Co-authored-by: Young <afe.young@gmail.com>
Co-authored-by: you-n-g <you-n-g@users.noreply.github.com>
Co-authored-by: Taozhi Wang <taozhi.mark.wang@gmail.com>
Co-authored-by: Suhan Cui <51844791+SH-Src@users.noreply.github.com>

rdagent/app/qlib_rd_loop/conf.py

@@ -35,8 +35,10 @@ class FactorBasePropSetting(BasePropSetting):
     # 2) sub task specific:
     origin_report_path: str = "data/report_origin"
     local_report_path: str = "data/report"
-    report_result_json_file_path: str = "git_ignore_folder/res_dict.json"
+    report_result_json_file_path: str = "git_ignore_folder/res_dict.csv"
     progress_file_path: str = "git_ignore_folder/progress.pkl"
+    report_extract_result: str = "git_ignore_folder/hypo_exp_cache.pkl"
+    max_factor_per_report: int = 10000
 
 
 FACTOR_PROP_SETTING = FactorBasePropSetting()

rdagent/app/qlib_rd_loop/factor_from_report_sh.py

@@ -1,8 +1,11 @@
 # TODO: we should have more advanced mechanism to handle such requirements for saving sessions.
+import csv
 import json
 import pickle
 from pathlib import Path
+from typing import Any
 
+import fire
 import pandas as pd
 from dotenv import load_dotenv
 from jinja2 import Environment, StrictUndefined
@@ -12,7 +15,10 @@ from rdagent.components.document_reader.document_reader import (
     extract_first_page_screenshot_from_pdf,
     load_and_process_pdfs_by_langchain,
 )
+from rdagent.components.workflow.conf import BasePropSetting
+from rdagent.components.workflow.rd_loop import RDLoop
 from rdagent.core.developer import Developer
+from rdagent.core.exception import FactorEmptyError
 from rdagent.core.prompts import Prompts
 from rdagent.core.proposal import (
     Hypothesis,
@@ -34,40 +40,16 @@ from rdagent.scenarios.qlib.factor_experiment_loader.pdf_loader import (
     FactorExperimentLoaderFromPDFfiles,
     classify_report_from_dict,
 )
+from rdagent.utils.workflow import LoopBase, LoopMeta
 
-assert load_dotenv()
-
-scen: Scenario = import_class(FACTOR_PROP_SETTING.scen)()
-
-hypothesis_gen: HypothesisGen = import_class(FACTOR_PROP_SETTING.hypothesis_gen)(scen)
-
-hypothesis2experiment: Hypothesis2Experiment = import_class(FACTOR_PROP_SETTING.hypothesis2experiment)()
-
-qlib_factor_coder: Developer = import_class(FACTOR_PROP_SETTING.coder)(scen)
-
-qlib_factor_runner: Developer = import_class(FACTOR_PROP_SETTING.runner)(scen)
-
-qlib_factor_summarizer: HypothesisExperiment2Feedback = import_class(FACTOR_PROP_SETTING.summarizer)(scen)
-
-with open(FACTOR_PROP_SETTING.report_result_json_file_path, "r") as f:
-    judge_pdf_data = json.load(f)
+with open(FACTOR_PROP_SETTING.report_result_json_file_path, "r") as input_file:
+    csv_reader = csv.reader(input_file)
+    judge_pdf_data = [row[0] for row in csv_reader]
 
 prompts_path = Path(__file__).parent / "prompts.yaml"
 prompts = Prompts(file_path=prompts_path)
 
 
-def save_progress(trace, current_index):
-    with open(FACTOR_PROP_SETTING.progress_file_path, "wb") as f:
-        pickle.dump((trace, current_index), f)
-
-
-def load_progress():
-    if Path(FACTOR_PROP_SETTING.progress_file_path).exists():
-        with open(FACTOR_PROP_SETTING.progress_file_path, "rb") as f:
-            return pickle.load(f)
-    return Trace(scen=scen), 0
-
-
 def generate_hypothesis(factor_result: dict, report_content: str) -> str:
     system_prompt = (
         Environment(undefined=StrictUndefined).from_string(prompts["hypothesis_generation"]["system"]).render()
@@ -123,52 +105,95 @@ def extract_factors_and_implement(report_file_path: str) -> tuple:
     return exp, hypothesis
 
 
-trace, start_index = load_progress()
+class FactorReportLoop(LoopBase, metaclass=LoopMeta):
+    skip_loop_error = (FactorEmptyError,)
 
-try:
-    judge_pdf_data_items = list(judge_pdf_data.items())
-    for index in range(start_index, len(judge_pdf_data_items)):
-        if index > 1000:
-            break
-        file_path, attributes = judge_pdf_data_items[index]
-        if attributes["class"] == 1:
-            report_file_path = Path(
-                file_path.replace(FACTOR_PROP_SETTING.origin_report_path, FACTOR_PROP_SETTING.local_report_path)
-            )
-            if report_file_path.exists():
-                logger.info(f"Processing {report_file_path}")
+    def __init__(self, PROP_SETTING: BasePropSetting):
+        scen: Scenario = import_class(PROP_SETTING.scen)()
 
-                with logger.tag("r"):
-                    exp, hypothesis = extract_factors_and_implement(str(report_file_path))
-                    if exp is None:
-                        continue
-                    exp.based_experiments = [t[1] for t in trace.hist if t[2]]
-                    if len(exp.based_experiments) == 0:
-                        exp.based_experiments.append(QlibFactorExperiment(sub_tasks=[]))
-                    logger.log_object(hypothesis, tag="hypothesis generation")
-                    logger.log_object(exp.sub_tasks, tag="experiment generation")
+        self.coder: Developer = import_class(PROP_SETTING.coder)(scen)
+        self.runner: Developer = import_class(PROP_SETTING.runner)(scen)
 
-                with logger.tag("d"):
-                    exp = qlib_factor_coder.develop(exp)
-                    logger.log_object(exp.sub_workspace_list)
+        self.summarizer: HypothesisExperiment2Feedback = import_class(PROP_SETTING.summarizer)(scen)
+        self.trace = Trace(scen=scen)
 
-                with logger.tag("ef"):
-                    exp = qlib_factor_runner.develop(exp)
-                    if exp is None:
-                        logger.error(f"Factor extraction failed for {report_file_path}. Skipping to the next report.")
-                        continue
-                    logger.log_object(exp, tag="factor runner result")
-                    feedback = qlib_factor_summarizer.generate_feedback(exp, hypothesis, trace)
-                    logger.log_object(feedback, tag="feedback")
+        self.judge_pdf_data_items = judge_pdf_data
+        self.index = 0
+        self.hypo_exp_cache = (
+            pickle.load(open(FACTOR_PROP_SETTING.report_extract_result, "rb"))
+            if Path(FACTOR_PROP_SETTING.report_extract_result).exists()
+            else {}
+        )
+        super().__init__()
 
-                trace.hist.append((hypothesis, exp, feedback))
-                logger.info(f"Processed {report_file_path}: Result: {exp}")
+    def propose_hypo_exp(self, prev_out: dict[str, Any]):
+        with logger.tag("r"):
+            while True:
+                if self.index > 100:
+                    break
+                report_file_path = self.judge_pdf_data_items[self.index]
+                self.index += 1
+                if report_file_path in self.hypo_exp_cache:
+                    hypothesis, exp = self.hypo_exp_cache[report_file_path]
+                    exp.based_experiments = [QlibFactorExperiment(sub_tasks=[])] + [
+                        t[1] for t in self.trace.hist if t[2]
+                    ]
+                else:
+                    continue
+                # else:
+                #     exp, hypothesis = extract_factors_and_implement(str(report_file_path))
+                #     if exp is None:
+                #         continue
+                #     exp.based_experiments = [QlibFactorExperiment(sub_tasks=[])] + [t[1] for t in self.trace.hist if t[2]]
+                #     self.hypo_exp_cache[report_file_path] = (hypothesis, exp)
+                #     pickle.dump(self.hypo_exp_cache, open(FACTOR_PROP_SETTING.report_extract_result, "wb"))
+                with logger.tag("extract_factors_and_implement"):
+                    with logger.tag("load_pdf_screenshot"):
+                        pdf_screenshot = extract_first_page_screenshot_from_pdf(report_file_path)
+                        logger.log_object(pdf_screenshot)
+                exp.sub_workspace_list = exp.sub_workspace_list[: FACTOR_PROP_SETTING.max_factor_per_report]
+                exp.sub_tasks = exp.sub_tasks[: FACTOR_PROP_SETTING.max_factor_per_report]
+                logger.log_object(hypothesis, tag="hypothesis generation")
+                logger.log_object(exp.sub_tasks, tag="experiment generation")
+                return hypothesis, exp
 
-                # Save progress after processing each report
-                save_progress(trace, index + 1)
-            else:
-                logger.error(f"File not found: {report_file_path}")
-except Exception as e:
-    logger.error(f"An error occurred: {e}")
-    save_progress(trace, index)
-    raise
+    def coding(self, prev_out: dict[str, Any]):
+        with logger.tag("d"):  # develop
+            exp = self.coder.develop(prev_out["propose_hypo_exp"][1])
+            logger.log_object(exp.sub_workspace_list, tag="coder result")
+        return exp
+
+    def running(self, prev_out: dict[str, Any]):
+        with logger.tag("ef"):  # evaluate and feedback
+            exp = self.runner.develop(prev_out["coding"])
+            if exp is None:
+                logger.error(f"Factor extraction failed.")
+                raise FactorEmptyError("Factor extraction failed.")
+            logger.log_object(exp, tag="runner result")
+        return exp
+
+    def feedback(self, prev_out: dict[str, Any]):
+        feedback = self.summarizer.generate_feedback(prev_out["running"], prev_out["propose_hypo_exp"][0], self.trace)
+        with logger.tag("ef"):  # evaluate and feedback
+            logger.log_object(feedback, tag="feedback")
+        self.trace.hist.append((prev_out["propose_hypo_exp"][0], prev_out["running"], feedback))
+
+
+def main(path=None, step_n=None):
+    """
+    You can continue running session by
+
+    .. code-block:: python
+
+        dotenv run -- python rdagent/app/qlib_rd_loop/factor_from_report_sh.py $LOG_PATH/__session__/1/0_propose  --step_n 1   # `step_n` is a optional paramter
+
+    """
+    if path is None:
+        model_loop = FactorReportLoop(FACTOR_PROP_SETTING)
+    else:
+        model_loop = FactorReportLoop.load(path)
+    model_loop.run(step_n=step_n)
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

rdagent/components/coder/factor_coder/CoSTEER/evaluators.py

@@ -165,19 +165,43 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
             )
             .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.")
         )
-        resp = APIBackend().build_messages_and_create_chat_completion(
-            user_prompt=gen_df_info_str, system_prompt=system_prompt, json_mode=True
-        )
-        resp_dict = json.loads(resp)
-        if isinstance(resp_dict["output_format_decision"], str) and resp_dict["output_format_decision"].lower() in (
-            "true",
-            "false",
-        ):
-            resp_dict["output_format_decision"] = bool(resp_dict["output_format_decision"])
-        return (
-            resp_dict["output_format_feedback"],
-            resp_dict["output_format_decision"],
-        )
+
+        # TODO: with retry_context(retry_n=3, except_list=[KeyError]):
+        max_attempts = 3
+        attempts = 0
+        final_evaluation_dict = None
+
+        while attempts < max_attempts:
+            try:
+                resp = APIBackend().build_messages_and_create_chat_completion(
+                    user_prompt=gen_df_info_str, system_prompt=system_prompt, json_mode=True
+                )
+                resp_dict = json.loads(resp)
+
+                if isinstance(resp_dict["output_format_decision"], str) and resp_dict[
+                    "output_format_decision"
+                ].lower() in (
+                    "true",
+                    "false",
+                ):
+                    resp_dict["output_format_decision"] = bool(resp_dict["output_format_decision"])
+
+                return (
+                    resp_dict["output_format_feedback"],
+                    resp_dict["output_format_decision"],
+                )
+
+            except json.JSONDecodeError as e:
+                raise ValueError("Failed to decode JSON response from API.") from e
+
+            except KeyError as e:
+                attempts += 1
+                if attempts >= max_attempts:
+                    raise KeyError(
+                        "Response from API is missing 'output_format_decision' or 'output_format_feedback' key after multiple attempts."
+                    ) from e
+
+        return "Failed to evaluate output format after multiple attempts.", False
 
 
 class FactorDatetimeDailyEvaluator(FactorEvaluator):

rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py

@@ -66,29 +66,27 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):
 
         # 2. 选择selection方法
         # if the number of factors to be implemented is larger than the limit, we need to select some of them
-        if FACTOR_IMPLEMENT_SETTINGS.select_ratio < 1:
-            # if the number of loops is equal to the select_loop, we need to select some of them
-            implementation_factors_per_round = round(
-                FACTOR_IMPLEMENT_SETTINGS.select_ratio * len(to_be_finished_task_index) + 0.5
-            )  # ceilling
-            implementation_factors_per_round = min(
-                implementation_factors_per_round, len(to_be_finished_task_index)
-            )  # but not exceed the total number of tasks
 
-            if FACTOR_IMPLEMENT_SETTINGS.select_method == "random":
-                to_be_finished_task_index = RandomSelect(
-                    to_be_finished_task_index,
-                    implementation_factors_per_round,
-                )
+        if FACTOR_IMPLEMENT_SETTINGS.select_threshold < len(to_be_finished_task_index):
+            # Select a fixed number of factors if the total exceeds the threshold
+            implementation_factors_per_round = FACTOR_IMPLEMENT_SETTINGS.select_threshold
+        else:
+            implementation_factors_per_round = len(to_be_finished_task_index)
 
-            if FACTOR_IMPLEMENT_SETTINGS.select_method == "scheduler":
-                to_be_finished_task_index = LLMSelect(
-                    to_be_finished_task_index,
-                    implementation_factors_per_round,
-                    evo,
-                    queried_knowledge.former_traces,
-                    self.scen,
-                )
+        if FACTOR_IMPLEMENT_SETTINGS.select_method == "random":
+            to_be_finished_task_index = RandomSelect(
+                to_be_finished_task_index,
+                implementation_factors_per_round,
+            )
+
+        if FACTOR_IMPLEMENT_SETTINGS.select_method == "scheduler":
+            to_be_finished_task_index = LLMSelect(
+                to_be_finished_task_index,
+                implementation_factors_per_round,
+                evo,
+                queried_knowledge.former_traces,
+                self.scen,
+            )
 
         result = multiprocessing_wrapper(
             [

rdagent/components/coder/factor_coder/config.py

@@ -39,7 +39,7 @@ class FactorImplementSettings(BaseSettings):
     file_based_execution_timeout: int = 120  # seconds for each factor implementation execution
 
     select_method: SELECT_METHOD = "random"
-    select_ratio: float = 0.5
+    select_threshold: int = 10
 
     max_loop: int = 10
 

rdagent/components/proposal/prompts.yaml

@@ -8,6 +8,7 @@ hypothesis_gen:
     Please generate the output following the format and specifications below:
     {{ hypothesis_output_format }}
     Here are the specifications: {{ hypothesis_specification }}
+
   user_prompt: |-
     The user has made several hypothesis on this scenario and did several evaluation on them.
     The former hypothesis and the corresponding feedbacks are as follows (focus on the last one & the new hypothesis that it provides and reasoning to see if you agree):

rdagent/scenarios/qlib/developer/factor_runner.py

@@ -104,6 +104,7 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]):
 
             # Sort and nest the combined factors under 'feature'
             combined_factors = combined_factors.sort_index()
+            combined_factors = combined_factors.loc[:, ~combined_factors.columns.duplicated(keep="last")]
             new_columns = pd.MultiIndex.from_product([["feature"], combined_factors.columns])
             combined_factors.columns = new_columns
 

rdagent/scenarios/qlib/experiment/factor_experiment.py

@@ -42,7 +42,56 @@ class QlibFactorScenario(Scenario):
 
     @property
     def rich_style_description(self) -> str:
-        return "Below is QlibFactor Evolving Automatic R&D Demo."
+        return """
+### Qlib Factor Evolving Automatic R&D Demo
+
+#### [Overview](#_summary)
+
+The demo showcases the iterative process of hypothesis generation, knowledge construction, and decision-making. It highlights how financial factors evolve through continuous feedback and refinement.
+
+#### Key Steps
+
+1. **Hypothesis Generation**
+   - Generate and propose initial hypotheses based on data and domain knowledge.
+
+2. **Factor Creation**
+   - Develop, define, and write new financial factors.
+   - Test these factors to gather empirical results.
+
+3. **Factor Validation**
+   - Validate the newly created factors quantitatively.
+
+4. **Backtesting with Qlib**
+   - **Dataset**: CSI300
+   - **Model**: LGBModel
+   - **Factors**: Alpha158 +
+   - **Data Split**:
+     - **Train**: 2008-01-01 to 2014-12-31
+     - **Valid**: 2015-01-01 to 2016-12-31
+     - **Test**: 2017-01-01 to 2020-08-01
+
+5. **Feedback Analysis**
+   - Analyze backtest results.
+   - Incorporate feedback to refine hypotheses.
+
+6. **Hypothesis Refinement**
+   - Refine hypotheses based on feedback and repeat the process.
+
+#### [Automated R&D](#_rdloops)
+
+- **[R (Research)](#_research)**
+  - Iteration of ideas and hypotheses.
+  - Continuous learning and knowledge construction.
+
+- **[D (Development)](#_development)*
+  - Evolving code generation and model refinement.
+  - Automated implementation and testing of financial factors.
+
+#### [Objective](#_summary)
+
+To demonstrate the dynamic evolution of financial factors through the Qlib platform, emphasizing how each iteration enhances the accuracy and reliability of the resulting financial factors.
+
+        """
 
     def get_scenario_all_desc(self) -> str:
         return f"""Background of the scenario:

rdagent/scenarios/qlib/prompts.yaml

@@ -48,72 +48,13 @@ model_hypothesis_specification: |-
     6th Round Hypothesis (If fourth round didn't work):  The model should be a CNN. The CNN should have 5 convolutional layers. Use Leaky ReLU activation for all layers. Use dropout regularization with a rate of 0.3. (Reasoning: As regularisation rate of 0.5 didn't work, we only change a new regularisation and keep the other elements that worked. This means making changes in the current level.)    
 
 factor_hypothesis_specification: |-
-  Specifications:
-    - Hypotheses should grow and evolve based on the previous hypothesis. If there is no previous hypothesis, start with something simple.
-    - Gradually build upon previous hypotheses and feedback.
-    - Ensure that the hypothesis focuses on the creation and selection of factors in quantitative finance.
-    - Each hypothesis should address specific factor characteristics such as type (momentum, value, quality), calculation methods, or inclusion criteria.
-    - Avoid hypotheses related to model architecture or optimization processes.
-    - If a hypothesis can be improved further, refine it. If it achieves the desired results, explore a new direction. Previous factors exceeding SOTA (State of the Art) are preserved and combined with new factors for subsequent evaluations.
-
-  Guiding Principles:
-  1. Diversity and Depth:
-    - Ensure a wide range of factor types, incorporating various financial dimensions (e.g., momentum, value, quality, volatility, sentiment).
-    - Explore different calculation methods and inclusion criteria to understand their impact.
-    - Consider combining multiple factors or filtering criteria for more sophisticated hypotheses.
-
-  2. Iterative Improvement:
-    - Build upon previous hypotheses, incorporating feedback and observed results.
-    - Aim for continuous refinement and complexity over iterations, starting from basic factors to more advanced combinations and techniques.
-
-  3. Contextual Relevance:
-    - Tailor hypotheses to the specific financial context and current market conditions.
-    - Leverage domain knowledge and recent financial research to inform hypothesis creation.
-
-  Sample Hypotheses (Use the format for guidance, not the specific content):
-  - "Include a momentum factor based on the last 12 months' returns."
-  - "Add a value factor calculated as the book-to-market ratio."
-  - "Incorporate a quality factor derived from return on equity (ROE)."
-  - "Use a volatility factor based on the standard deviation of returns over the past 6 months."
-  - "Include a sentiment factor derived from news sentiment scores."
-  - "The momentum factor should be calculated using a 6-month look-back period."
-  - "Combine value and momentum factors using a weighted average approach."
-  - "Filter stocks by market capitalization before calculating the factors."
-  - "Explore a liquidity factor based on the trading volume and bid-ask spread."
-  - "Investigate the impact of an earnings surprise factor calculated from recent earnings announcements."
-  - "Develop a composite factor integrating ESG (Environmental, Social, Governance) scores with traditional financial metrics."
-
-  Detailed Workflow: (These are just examples for the format, do not be constrained by these examples)
-  1. Initial Hypothesis:
-    - Begin with a simple factor, such as "Include a momentum factor based on the last 12 months' returns."
-
-  2. Refine Hypothesis:
-    - If the initial hypothesis is promising, refine it further, e.g., "The momentum factor should be calculated using a 6-month look-back period."
-
-  3. Combine Factors:
-    - As individual factors show potential, combine them, e.g., "Combine value and momentum factors using a weighted average approach."
-
-  4. Contextual Adjustments:
-    - Adjust factors based on market conditions or new financial insights, e.g., "Incorporate a quality factor derived from return on equity (ROE)."
-
-  5. Advanced Hypotheses:
-    - Explore sophisticated combinations or new types of factors, e.g., "Develop a composite factor integrating ESG scores with traditional financial metrics."
-
-  Important Note:
-    Logic Explanation:
-      - If the previous hypothesis factor exceeds SOTA, the SOTA factor library will include this factor.
-      - The new experiment will generate new factors, which will be combined with the factors in the SOTA library.
-      - These combined factors will be backtested and compared against the current SOTA to iterate continuously.
-    Development Directions:
-      - New Direction:
-          - Propose a new factor direction for exploration and construction.
-      - Optimization of Existing Direction:
-          - If the previous experiment's factor replaced SOTA, you can further improve upon that factor.
-          - Clearly specify the differences in name and improvements compared to the previous factor.
-      - Continued Research:
-          - If the previous experiment's factor did not replace SOTA, continue researching how to optimize and construct factors in this direction.
-    Final Goal:
-      - The final maintained SOTA should be the continuous accumulation of factors that surpass each iteration.
+    Focus on the type of factor and the financial trends indicated by the factor. Try to explain from a broader perspective. 
+    If you think some aspects are not necessary, you can omit them.
+    You can start with simple and highly likely effective factors, then gradually move towards more complex ones.
+    Additionally, if you feel a new direction is needed, you don't need to implement the previous factors again, as the factors 
+    that previously surpassed SOTA are already in the factor library and will be included in each run. Therefore, your new direction can completely avoid the previous factors. 
+    It's preferable to provide an explanation for why you are shifting to a new direction, based on financial principles, economic theories, or other reasons. 
+    If a suitable explanation is not available, it's okay not to provide one.
 
 factor_experiment_output_format: |-
   The output should follow JSON format. The schema is as follows:
@@ -165,18 +106,35 @@ model_experiment_output_format: |-
 
 factor_feedback_generation:
   system: |-
-    You are a professional result analysis assistant in data-driven R&D.
+    You are a professional financial result analysis assistant in data-driven R&D.
     The task is described in the following scenario:
     {{ scenario }}
     You will receive a hypothesis, multiple tasks with their factors, and some results.
-    Your feedback should specify whether the current result supports or refutes the hypothesis, compare it with previous SOTA results, and suggest improvements or new directions.
-    Please provide detailed and constructive feedback for the future exploration.
-    Please respond in JSON format, and example JSON Structure for Result Analysis:
+    Your feedback should specify whether the current result supports or refutes the hypothesis, compare it with previous SOTA (State of the Art) results, and suggest improvements or new directions.
+    
+    Please consider the following points:
+      Logic Explanation:
+          - If the previous hypothesis factor surpasses the SOTA, include this factor in the SOTA factor library.
+          - New experiments will generate new factors, which will be combined with the factors in the SOTA library.
+          - These combined factors will be backtested and compared against the current SOTA to continuously iterate.
+        Development Directions:
+          - New Direction:
+              - Propose a new factor direction for exploration and development.
+          - Optimization of Existing Direction:
+              - If the previous experiment's factor replaced the SOTA, suggest further improvements to that factor.
+              - Clearly specify the differences in name and improvements compared to the previous factor.
+          - Continued Research:
+              - If the previous experiment's factor did not replace the SOTA, suggest ways to optimize and develop factors in this direction.
+        Final Goal:
+          - The ultimate goal is to continuously accumulate factors that surpass each iteration to maintain the best SOTA.
+    
+    Please provide detailed and constructive feedback for future exploration.
+    Respond in JSON format. Example JSON structure for Result Analysis:
     {
       "Observations": "Your overall observations here",
       "Feedback for Hypothesis": "Observations related to the hypothesis",
-      "New Hypothesis": "Put your new hypothesis here.",
-      "Reasoning": "Provide reasoning for the hypothesis here.",
+      "New Hypothesis": "Your new hypothesis here",
+      "Reasoning": "Reasoning for the new hypothesis",
       "Replace Best Result": "yes or no"
     }
   user: |-
@@ -203,14 +161,16 @@ factor_feedback_generation:
     - 1day.excess_return_with_cost.information_ratio: Evaluates the excess return per unit of risk considering transaction costs.
 
     When judging the results:
-    1. Prioritize metrics that consider transaction costs (with cost):
-    - These metrics provide a more accurate representation of real-world performance.
+    1. Prioritize metrics that not consider transaction costs (without cost):
     2. Evaluate all metrics:
         - Compare the combined results against the current best results across all metrics to get a comprehensive view of performance.
     3. Focus on the annualized return considering transaction costs:
         - This metric is particularly important as it gives a clear picture of long-term profitability.
     4. Recommendation for replacement:
         - If the new factor demonstrates a significant improvement in the annualized return considering transaction costs, it should be recommended to replace the current best result, even if other metrics show minor variations.
+    5. Consider changing direction if there is a significant gap with SOTA:
+        - If the new results differ significantly from the SOTA, it may be necessary to explore a new direction. 
+        - In this case, previous factors should not be implemented again as the factors that surpassed SOTA are already in the factor library and will be included in each run.
 
     Please provide detailed feedback and recommend whether to replace the best result if the new factor proves superior.
 

rdagent/scenarios/qlib/proposal/factor_proposal.py

@@ -87,10 +87,7 @@ class QlibFactorHypothesis2Experiment(FactorHypothesis2Experiment):
             tasks.append(FactorTask(factor_name, description, formulation, variables))
 
         exp = QlibFactorExperiment(tasks)
-        exp.based_experiments = [t[1] for t in trace.hist if t[2]]
-
-        if len(exp.based_experiments) == 0:
-            exp.based_experiments.append(QlibFactorExperiment(sub_tasks=[]))
+        exp.based_experiments = [QlibFactorExperiment(sub_tasks=[])] + [t[1] for t in trace.hist if t[2]]
 
         unique_tasks = []