feat: add RD-Agent-Quant scenario (#838)

* fix model input shape bug and costeer_model bug * fix a bug * fix a bug in docker result extraction * a system-level optimization * add a filter of stdout * update * add stdout to model * model training_hyperparameters update * quant scenario * update some quant settings * llm choose action * Thompson Sampling Bandit for action choosing * refine both scens * add trace messages for quant scen * fix some bugs * fix some bugs * update * update * update * fix * fix * fix * update for merge * fix ci * fix some bugs * fix ci * fix ci * fix ci * fix ci * refactor * default qlib4rdagent local env downloading * fix ci * fix ci * fix a bug * fix ci * fix: align all prompts on template (#908) * use template to render all prompts * fix CI --------- Co-authored-by: Xu Yang <xuyang1@microsoft.com> * add fin_quant in cli * fix a bug * fix ci * fix some bugs * refactor * remove the columns in hypothesis if no value generated in this column * fix a bug * fix ci * fix conda env * add qlib gitignore * remove existed qlib folder & install torch in qlib conda * fix workspace ui in feedback * align model config in coder and runner in docker or conda * fix CI * fix CI --------- Co-authored-by: Xu Yang <peteryang@vip.qq.com> Co-authored-by: Xu Yang <xuyang1@microsoft.com>
2026-07-29 08:27:43 +00:00 · 2025-05-29 16:16:51 +08:00
parent b0e88c7375
commit d1019cb568
64 changed files with 2427 additions and 1025 deletions
@@ -15,7 +15,7 @@ class FactorCoSTEERSettings(CoSTEERSettings):
    simple_background: bool = False
    """Whether to use simple background information for code feedback"""

-    file_based_execution_timeout: int = 120
+    file_based_execution_timeout: int = 3600
    """Timeout in seconds for each factor implementation execution"""

    select_method: str = "random"
@@ -1,20 +1,16 @@
 import io
 import json
 from abc import abstractmethod
-from pathlib import Path
 from typing import Dict, Tuple

 import pandas as pd
-from jinja2 import Environment, StrictUndefined

 from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
 from rdagent.components.coder.factor_coder.factor import FactorTask
 from rdagent.core.experiment import Task, Workspace
-from rdagent.core.prompts import Prompts
 from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend
-
-evaluate_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
+from rdagent.utils.agent.tpl import T


 class FactorEvaluator:
@@ -81,36 +77,26 @@ class FactorCodeEvaluator(FactorEvaluator):
        factor_information = target_task.get_task_information()
        code = implementation.all_codes

-        system_prompt = (
-            Environment(undefined=StrictUndefined)
-            .from_string(evaluate_prompts["evaluator_code_feedback_v1_system"])
-            .render(
-                scenario=(
-                    self.scen.get_scenario_all_desc(
-                        target_task,
-                        filtered_tag="feature",
-                        simple_background=FACTOR_COSTEER_SETTINGS.simple_background,
-                    )
-                    if self.scen is not None
-                    else "No scenario description."
+        system_prompt = T(".prompts:evaluator_code_feedback_v1_system").r(
+            scenario=(
+                self.scen.get_scenario_all_desc(
+                    target_task,
+                    filtered_tag="feature",
+                    simple_background=FACTOR_COSTEER_SETTINGS.simple_background,
                )
+                if self.scen is not None
+                else "No scenario description."
            )
        )

        execution_feedback_to_render = execution_feedback
        for _ in range(10):  # 10 times to split the content is enough
-            user_prompt = (
-                Environment(undefined=StrictUndefined)
-                .from_string(
-                    evaluate_prompts["evaluator_code_feedback_v1_user"],
-                )
-                .render(
-                    factor_information=factor_information,
-                    code=code,
-                    execution_feedback=execution_feedback_to_render,
-                    value_feedback=value_feedback,
-                    gt_code=gt_implementation.code if gt_implementation else None,
-                )
+            user_prompt = T(".prompts:evaluator_code_feedback_v1_user").r(
+                factor_information=factor_information,
+                code=code,
+                execution_feedback=execution_feedback_to_render,
+                value_feedback=value_feedback,
+                gt_code=gt_implementation.code if gt_implementation else None,
            )
            if (
                APIBackend().build_messages_and_calculate_token(
@@ -189,17 +175,11 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
        buffer = io.StringIO()
        gen_df.info(buf=buffer)
        gen_df_info_str = f"The user is currently working on a feature related task.\nThe output dataframe info is:\n{buffer.getvalue()}"
-        system_prompt = (
-            Environment(undefined=StrictUndefined)
-            .from_string(
-                evaluate_prompts["evaluator_output_format_system"],
-            )
-            .render(
-                scenario=(
-                    self.scen.get_scenario_all_desc(implementation.target_task, filtered_tag="feature")
-                    if self.scen is not None
-                    else "No scenario description."
-                )
+        system_prompt = T(".prompts:evaluator_output_format_system").r(
+            scenario=(
+                self.scen.get_scenario_all_desc(implementation.target_task, filtered_tag="feature")
+                if self.scen is not None
+                else "No scenario description."
            )
        )

@@ -504,35 +484,25 @@ class FactorFinalDecisionEvaluator(FactorEvaluator):
        code_feedback: str,
        **kwargs,
    ) -> Tuple:
-        system_prompt = (
-            Environment(undefined=StrictUndefined)
-            .from_string(evaluate_prompts["evaluator_final_decision_v1_system"])
-            .render(
-                scenario=(
-                    self.scen.get_scenario_all_desc(target_task, filtered_tag="feature")
-                    if self.scen is not None
-                    else "No scenario description."
-                )
+        system_prompt = T(".prompts:evaluator_final_decision_v1_system").r(
+            scenario=(
+                self.scen.get_scenario_all_desc(target_task, filtered_tag="feature")
+                if self.scen is not None
+                else "No scenario description."
            )
        )
        execution_feedback_to_render = execution_feedback

        for _ in range(10):  # 10 times to split the content is enough
-            user_prompt = (
-                Environment(undefined=StrictUndefined)
-                .from_string(
-                    evaluate_prompts["evaluator_final_decision_v1_user"],
-                )
-                .render(
-                    factor_information=target_task.get_task_information(),
-                    execution_feedback=execution_feedback_to_render,
-                    code_feedback=code_feedback,
-                    value_feedback=(
-                        value_feedback
-                        if value_feedback is not None
-                        else "No Ground Truth Value provided, so no evaluation on value is performed."
-                    ),
-                )
+            user_prompt = T(".prompts:evaluator_final_decision_v1_user").r(
+                factor_information=target_task.get_task_information(),
+                execution_feedback=execution_feedback_to_render,
+                code_feedback=code_feedback,
+                value_feedback=(
+                    value_feedback
+                    if value_feedback is not None
+                    else "No Ground Truth Value provided, so no evaluation on value is performed."
+                ),
            )
            if (
                APIBackend().build_messages_and_calculate_token(
@@ -1,11 +1,8 @@
 from __future__ import annotations

 import json
-from pathlib import Path
 from typing import Dict

-from jinja2 import Environment, StrictUndefined
-
 from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedback
 from rdagent.components.coder.CoSTEER.evolving_strategy import (
    MultiProcessEvolvingStrategy,
@@ -17,11 +14,9 @@ from rdagent.components.coder.CoSTEER.knowledge_management import (
 from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
 from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace, FactorTask
 from rdagent.core.experiment import FBWorkspace
-from rdagent.core.prompts import Prompts
 from rdagent.oai.llm_conf import LLM_SETTINGS
 from rdagent.oai.llm_utils import APIBackend
-
-implement_prompts = Prompts(file_path=Path(__file__).parent / "prompts.yaml")
+from rdagent.utils.agent.tpl import T


 class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
@@ -36,24 +31,14 @@ class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
        queried_former_failed_knowledge_to_render: list,
        queried_similar_error_knowledge_to_render: list,
    ) -> str:
-        error_summary_system_prompt = (
-            Environment(undefined=StrictUndefined)
-            .from_string(implement_prompts["evolving_strategy_error_summary_v2_system"])
-            .render(
-                scenario=self.scen.get_scenario_all_desc(target_task),
-                factor_information_str=target_task.get_task_information(),
-                code_and_feedback=queried_former_failed_knowledge_to_render[-1].get_implementation_and_feedback_str(),
-            )
-            .strip("\n")
+        error_summary_system_prompt = T(".prompts:evolving_strategy_error_summary_v2_system").r(
+            scenario=self.scen.get_scenario_all_desc(target_task),
+            factor_information_str=target_task.get_task_information(),
+            code_and_feedback=queried_former_failed_knowledge_to_render[-1].get_implementation_and_feedback_str(),
        )
        for _ in range(10):  # max attempt to reduce the length of error_summary_user_prompt
-            error_summary_user_prompt = (
-                Environment(undefined=StrictUndefined)
-                .from_string(implement_prompts["evolving_strategy_error_summary_v2_user"])
-                .render(
-                    queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
-                )
-                .strip("\n")
+            error_summary_user_prompt = T(".prompts:evolving_strategy_error_summary_v2_user").r(
+                queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
            )
            if (
                APIBackend().build_messages_and_calculate_token(
@@ -106,16 +91,9 @@ class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
        latest_attempt_to_latest_successful_execution = queried_knowledge.task_to_former_failed_traces[
            target_factor_task_information
        ][1]
-
-        system_prompt = (
-            Environment(undefined=StrictUndefined)
-            .from_string(
-                implement_prompts["evolving_strategy_factor_implementation_v1_system"],
-            )
-            .render(
-                scenario=self.scen.get_scenario_all_desc(target_task, filtered_tag="feature"),
-                queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
-            )
+        system_prompt = T(".prompts:evolving_strategy_factor_implementation_v1_system").r(
+            scenario=self.scen.get_scenario_all_desc(target_task, filtered_tag="feature"),
+            queried_former_failed_knowledge=queried_former_failed_knowledge_to_render,
        )
        queried_similar_successful_knowledge_to_render = queried_similar_successful_knowledge
        queried_similar_error_knowledge_to_render = queried_similar_error_knowledge
@@ -136,19 +114,12 @@ class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
            else:
                error_summary_critics = None
            # 构建user_prompt。开始写代码
-            user_prompt = (
-                Environment(undefined=StrictUndefined)
-                .from_string(
-                    implement_prompts["evolving_strategy_factor_implementation_v2_user"],
-                )
-                .render(
-                    factor_information_str=target_factor_task_information,
-                    queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render,
-                    queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
-                    error_summary_critics=error_summary_critics,
-                    latest_attempt_to_latest_successful_execution=latest_attempt_to_latest_successful_execution,
-                )
-                .strip("\n")
+            user_prompt = T(".prompts:evolving_strategy_factor_implementation_v2_user").r(
+                factor_information_str=target_factor_task_information,
+                queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render,
+                queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
+                error_summary_critics=error_summary_critics,
+                latest_attempt_to_latest_successful_execution=latest_attempt_to_latest_successful_execution,
            )
            if (
                APIBackend().build_messages_and_calculate_token(user_prompt=user_prompt, system_prompt=system_prompt)
@@ -49,6 +49,12 @@ class FactorTask(CoSTEERTask):
        return f"""factor_name: {self.factor_name}
 factor_description: {self.factor_description}
 factor_formulation: {self.factor_formulation}
+variables: {str(self.variables)}"""
+
+    def get_task_brief_information(self):
+        return f"""factor_name: {self.factor_name}
+factor_description: {self.factor_description}
+factor_formulation: {self.factor_formulation}
 variables: {str(self.variables)}"""

    def get_task_information_and_implementation_result(self):
@@ -116,6 +116,9 @@ evolving_strategy_error_summary_v2_system: |-

  You suggestion should not include any code, just some clear and short suggestions. Please point out very critical issues in your response, ignore non-important issues to avoid confusion. If no big issue found in the code, you can response "No critics found".

+  [NOTE]
+  1. When processing data, avoid time leakage.
+
  Please response the critic in the following format. Here is an example structure for the output:
  critic 1: The critic message to critic 1
  critic 2: The critic message to critic 2