fix: Comprehensive update to factor extraction. (#143)

* Init todo * update all code * update * Extract factors from financial reports loop finished * Fix two small bugs. * Delete rdagent/app/qlib_rd_loop/run_script.sh * Minor mod * Delete rdagent/app/qlib_rd_loop/nohup.out * Fix a small bug in file reading. * some updates * Update the detailed process and prompt of factor loop. * Evaluation & dataset * Optimize the prompt for generating hypotheses and feedback in the factor loop. * Generate new data * dataset generation * Performed further optimizations on the factor loop and report extraction loop, added log handling for both processes, and implemented a screenshot feature for report extraction. * Update rdagent/components/coder/factor_coder/CoSTEER/evaluators.py * Update package.txt for fitz. * add the result * Performed further optimizations on the factor loop and report extraction loop, added log handling for both processes, and implemented a screenshot feature for report extraction. (#100) (#102) - Performed further optimizations on the factor loop and report extraction loop. - Added log handling for both processes. - Implemented a screenshot feature for report extraction. * Analysis * Optimized log output. * Factor update * A draft of the "Quick Start" section for README * Add scenario descriptions. * Updates * Adjust content * Enable logging of backtesting in Qlib and store rich-text descriptions in Trace. Support one-step debugging for factor extraction. * Reformat analysis.py * CI fix * Refactor * remove useless code * fix bugs (#111) * Fix two small bugs. * Fix a merge bug. * Fix two small bugs. * fix some bugs. * Fix some format bugs. * Restore a file. * Fix a format bug. * draft renew of evaluators * fix a small bug. * fix a small bug * Support Factor Report Loop * Update framework for extracting factors from research reports. * Refactor report-based factor extraction and fix minor bugs. * fix a small bug of log. * change some prompts * improve factor_runner * fix a small bug * change some prompts * cancel some comments * cancel some comments and fix some bugs --------- Co-authored-by: Young <afe.young@gmail.com> Co-authored-by: you-n-g <you-n-g@users.noreply.github.com> Co-authored-by: Taozhi Wang <taozhi.mark.wang@gmail.com> Co-authored-by: Suhan Cui <51844791+SH-Src@users.noreply.github.com>
2026-07-28 07:57:44 +00:00 · 2024-08-02 15:04:49 +08:00
parent 515fb50ce2
commit 363f616aae
10 changed files with 243 additions and 186 deletions
@@ -165,19 +165,43 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
            )
            .render(scenario=self.scen.get_scenario_all_desc() if self.scen is not None else "No scenario description.")
        )
-        resp = APIBackend().build_messages_and_create_chat_completion(
-            user_prompt=gen_df_info_str, system_prompt=system_prompt, json_mode=True
-        )
-        resp_dict = json.loads(resp)
-        if isinstance(resp_dict["output_format_decision"], str) and resp_dict["output_format_decision"].lower() in (
-            "true",
-            "false",
-        ):
-            resp_dict["output_format_decision"] = bool(resp_dict["output_format_decision"])
-        return (
-            resp_dict["output_format_feedback"],
-            resp_dict["output_format_decision"],
-        )
+
+        # TODO: with retry_context(retry_n=3, except_list=[KeyError]):
+        max_attempts = 3
+        attempts = 0
+        final_evaluation_dict = None
+
+        while attempts < max_attempts:
+            try:
+                resp = APIBackend().build_messages_and_create_chat_completion(
+                    user_prompt=gen_df_info_str, system_prompt=system_prompt, json_mode=True
+                )
+                resp_dict = json.loads(resp)
+
+                if isinstance(resp_dict["output_format_decision"], str) and resp_dict[
+                    "output_format_decision"
+                ].lower() in (
+                    "true",
+                    "false",
+                ):
+                    resp_dict["output_format_decision"] = bool(resp_dict["output_format_decision"])
+
+                return (
+                    resp_dict["output_format_feedback"],
+                    resp_dict["output_format_decision"],
+                )
+
+            except json.JSONDecodeError as e:
+                raise ValueError("Failed to decode JSON response from API.") from e
+
+            except KeyError as e:
+                attempts += 1
+                if attempts >= max_attempts:
+                    raise KeyError(
+                        "Response from API is missing 'output_format_decision' or 'output_format_feedback' key after multiple attempts."
+                    ) from e
+
+        return "Failed to evaluate output format after multiple attempts.", False


 class FactorDatetimeDailyEvaluator(FactorEvaluator):
@@ -66,29 +66,27 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy):

        # 2. 选择selection方法
        # if the number of factors to be implemented is larger than the limit, we need to select some of them
-        if FACTOR_IMPLEMENT_SETTINGS.select_ratio < 1:
-            # if the number of loops is equal to the select_loop, we need to select some of them
-            implementation_factors_per_round = round(
-                FACTOR_IMPLEMENT_SETTINGS.select_ratio * len(to_be_finished_task_index) + 0.5
-            )  # ceilling
-            implementation_factors_per_round = min(
-                implementation_factors_per_round, len(to_be_finished_task_index)
-            )  # but not exceed the total number of tasks

-            if FACTOR_IMPLEMENT_SETTINGS.select_method == "random":
-                to_be_finished_task_index = RandomSelect(
-                    to_be_finished_task_index,
-                    implementation_factors_per_round,
-                )
+        if FACTOR_IMPLEMENT_SETTINGS.select_threshold < len(to_be_finished_task_index):
+            # Select a fixed number of factors if the total exceeds the threshold
+            implementation_factors_per_round = FACTOR_IMPLEMENT_SETTINGS.select_threshold
+        else:
+            implementation_factors_per_round = len(to_be_finished_task_index)

-            if FACTOR_IMPLEMENT_SETTINGS.select_method == "scheduler":
-                to_be_finished_task_index = LLMSelect(
-                    to_be_finished_task_index,
-                    implementation_factors_per_round,
-                    evo,
-                    queried_knowledge.former_traces,
-                    self.scen,
-                )
+        if FACTOR_IMPLEMENT_SETTINGS.select_method == "random":
+            to_be_finished_task_index = RandomSelect(
+                to_be_finished_task_index,
+                implementation_factors_per_round,
+            )
+
+        if FACTOR_IMPLEMENT_SETTINGS.select_method == "scheduler":
+            to_be_finished_task_index = LLMSelect(
+                to_be_finished_task_index,
+                implementation_factors_per_round,
+                evo,
+                queried_knowledge.former_traces,
+                self.scen,
+            )

        result = multiprocessing_wrapper(
            [
@@ -39,7 +39,7 @@ class FactorImplementSettings(BaseSettings):
    file_based_execution_timeout: int = 120  # seconds for each factor implementation execution

    select_method: SELECT_METHOD = "random"
-    select_ratio: float = 0.5
+    select_threshold: int = 10

    max_loop: int = 10