From a6ddf08e2a661eff9127d8bae2c69ef5ded54fd9 Mon Sep 17 00:00:00 2001 From: Xu Yang Date: Thu, 10 Oct 2024 20:29:17 +0800 Subject: [PATCH] perf: some small upgrade to factor costeer to improve the performance (#420) * 1. use dataframe.info instead of head 2. in former trace query, add the latest attempt to the last success execution * fix CI --- .../factor_coder/CoSTEER/evolving_strategy.py | 9 ++++++- .../CoSTEER/knowledge_management.py | 26 +++++++++++++++---- .../coder/factor_coder/CoSTEER/scheduler.py | 2 +- .../components/coder/factor_coder/config.py | 1 + .../coder/factor_coder/prompts.yaml | 7 +++++ rdagent/scenarios/qlib/experiment/utils.py | 8 ++++-- 6 files changed, 44 insertions(+), 9 deletions(-) diff --git a/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py b/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py index ac72545a..e165a5cd 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/evolving_strategy.py @@ -216,11 +216,17 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): ) # A dict, {{error_type:[[error_imp_knowledge, success_imp_knowledge],...]},...} queried_former_failed_knowledge = ( - queried_knowledge.former_traces[target_factor_task_information] if queried_knowledge is not None else [] + queried_knowledge.former_traces[target_factor_task_information][0] + if queried_knowledge is not None + else [] ) queried_former_failed_knowledge_to_render = queried_former_failed_knowledge + latest_attempt_to_latest_successful_execution = queried_knowledge.former_traces[ + target_factor_task_information + ][1] + system_prompt = ( Environment(undefined=StrictUndefined) .from_string( @@ -296,6 +302,7 @@ class FactorEvolvingStrategyWithGraph(MultiProcessEvolvingStrategy): queried_similar_error_knowledge=queried_similar_error_knowledge_to_render, error_summary=error_summary, error_summary_critics=error_summary_critics, + latest_attempt_to_latest_successful_execution=latest_attempt_to_latest_successful_execution, ) .strip("\n") ) diff --git a/rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py b/rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py index 36a7029f..80d311d3 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/knowledge_management.py @@ -296,6 +296,7 @@ class FactorGraphRAGStrategy(RAGStrategy): evo, factor_implementation_queried_graph_knowledge, FACTOR_IMPLEMENT_SETTINGS.v2_query_former_trace_limit, + FACTOR_IMPLEMENT_SETTINGS.v2_add_fail_attempt_to_latest_successful_execution, ) factor_implementation_queried_graph_knowledge = self.component_query( evo, @@ -392,6 +393,7 @@ class FactorGraphRAGStrategy(RAGStrategy): evo: EvolvableSubjects, factor_implementation_queried_graph_knowledge: FactorQueriedGraphKnowledge, v2_query_former_trace_limit: int = 5, + v2_add_fail_attempt_to_latest_successful_execution: bool = False, ) -> Union[QueriedKnowledge, set]: """ Query the former trace knowledge of the working trace, and find all the failed task information which tried more than fail_task_trial_limit times @@ -429,11 +431,25 @@ class FactorGraphRAGStrategy(RAGStrategy): else: current_index += 1 - factor_implementation_queried_graph_knowledge.former_traces[ - target_factor_task_information - ] = former_trace_knowledge[-v2_query_former_trace_limit:] + latest_attempt = None + if v2_add_fail_attempt_to_latest_successful_execution: + # When the last successful execution is not the last one in the working trace, it means we have tried to correct it. We should tell the agent this fail trial to avoid endless loop in the future. + if ( + len(former_trace_knowledge) > 0 + and len(self.knowledgebase.working_trace_knowledge[target_factor_task_information]) > 1 + and self.knowledgebase.working_trace_knowledge[target_factor_task_information].index( + former_trace_knowledge[-1] + ) + < len(self.knowledgebase.working_trace_knowledge[target_factor_task_information]) - 1 + ): + latest_attempt = self.knowledgebase.working_trace_knowledge[target_factor_task_information][-1] + + factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = ( + former_trace_knowledge[-v2_query_former_trace_limit:], + latest_attempt, + ) else: - factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = [] + factor_implementation_queried_graph_knowledge.former_traces[target_factor_task_information] = ([], None) return factor_implementation_queried_graph_knowledge @@ -607,7 +623,7 @@ class FactorGraphRAGStrategy(RAGStrategy): ): queried_last_trace = factor_implementation_queried_graph_knowledge.former_traces[ target_factor_task_information - ][-1] + ][0][-1] target_index = self.knowledgebase.working_trace_knowledge[target_factor_task_information].index( queried_last_trace, ) diff --git a/rdagent/components/coder/factor_coder/CoSTEER/scheduler.py b/rdagent/components/coder/factor_coder/CoSTEER/scheduler.py index 4e31e566..ecbdf1a4 100644 --- a/rdagent/components/coder/factor_coder/CoSTEER/scheduler.py +++ b/rdagent/components/coder/factor_coder/CoSTEER/scheduler.py @@ -40,7 +40,7 @@ def LLMSelect( # find corresponding former trace for each task target_factor_task_information = evo.sub_tasks[i].get_task_information() if target_factor_task_information in former_trace: - tasks.append((i, evo.sub_tasks[i], former_trace[target_factor_task_information])) + tasks.append((i, evo.sub_tasks[i], former_trace[target_factor_task_information][0])) system_prompt = ( Environment(undefined=StrictUndefined) diff --git a/rdagent/components/coder/factor_coder/config.py b/rdagent/components/coder/factor_coder/config.py index 5b573a78..702b304c 100644 --- a/rdagent/components/coder/factor_coder/config.py +++ b/rdagent/components/coder/factor_coder/config.py @@ -37,6 +37,7 @@ class FactorImplementSettings(BaseSettings): v2_query_component_limit: int = 1 v2_query_error_limit: int = 1 v2_query_former_trace_limit: int = 1 + v2_add_fail_attempt_to_latest_successful_execution: bool = False v2_error_summary: bool = False v2_knowledge_sampler: float = 1.0 diff --git a/rdagent/components/coder/factor_coder/prompts.yaml b/rdagent/components/coder/factor_coder/prompts.yaml index a3423ace..53d571ee 100644 --- a/rdagent/components/coder/factor_coder/prompts.yaml +++ b/rdagent/components/coder/factor_coder/prompts.yaml @@ -118,6 +118,13 @@ evolving_strategy_factor_implementation_v2_user: |- {{ similar_component_knowledge.implementation.code }} {% endfor %} {% endif %} + {% if latest_attempt_to_latest_successful_execution is not none %} + You have tried to correct your former failed code but still met some errors. Here is the latest attempt to the latest successful execution, try not to get the same error to your new code: + =====Your latest attempt===== + {{ latest_attempt_to_latest_successful_execution.implementation.code }} + =====Feedback to your latest attempt===== + {{ latest_attempt_to_latest_successful_execution.feedback }} + {% endif %} evolving_strategy_error_summary_v2_system: |- User is trying to implement some factors in the following scenario: diff --git a/rdagent/scenarios/qlib/experiment/utils.py b/rdagent/scenarios/qlib/experiment/utils.py index 5b81505b..91dbebfa 100644 --- a/rdagent/scenarios/qlib/experiment/utils.py +++ b/rdagent/scenarios/qlib/experiment/utils.py @@ -1,3 +1,4 @@ +import io import re import shutil from pathlib import Path @@ -81,10 +82,13 @@ def get_file_desc(p: Path) -> str: pd.set_option("display.max_columns", None) # or 1000 pd.set_option("display.max_rows", None) # or 1000 pd.set_option("display.max_colwidth", None) # or 199 + + buffer = io.StringIO() + df.info(verbose=True, buf=buffer, show_counts=False) return JJ_TPL.render( file_name=p.name, - type_desc="generated by `pd.read_hdf(filename).head()`", - content=df.head().to_string(), + type_desc="generated by `df.info(verbose=True, show_counts=False)`", + content=buffer.getvalue(), ) elif p.name.endswith(".md"): with open(p) as f: