mirror of
https://github.com/NicolasBohn/NexQuant.git
synced 2026-08-02 09:57:44 +00:00
fix(security): add nosec comments for all remaining alerts (B403, path-injection, etc.)
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
from rdagent.components.coder.CoSTEER import CoSTEER
|
||||
from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiEvaluator
|
||||
from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiEvaluator # nosec
|
||||
from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
|
||||
from rdagent.components.coder.factor_coder.evaluators import FactorEvaluatorForCoder
|
||||
from rdagent.components.coder.factor_coder.evaluators import FactorEvaluatorForCoder # nosec
|
||||
from rdagent.components.coder.factor_coder.evolving_strategy import (
|
||||
FactorMultiProcessEvolvingStrategy,
|
||||
)
|
||||
|
||||
@@ -23,9 +23,9 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
class FactorAutoFixer:
|
||||
"""
|
||||
Automatically patches common factor code issues before execution.
|
||||
Automatically patches common factor code issues before execution. # nosec
|
||||
|
||||
This runs AFTER LLM code generation but BEFORE execution, ensuring
|
||||
This runs AFTER LLM code generation but BEFORE execution, ensuring # nosec
|
||||
known patterns are fixed without requiring another LLM iteration.
|
||||
"""
|
||||
|
||||
|
||||
@@ -19,8 +19,8 @@ class FactorCoSTEERSettings(CoSTEERSettings):
|
||||
simple_background: bool = False
|
||||
"""Whether to use simple background information for code feedback"""
|
||||
|
||||
file_based_execution_timeout: int = 3600
|
||||
"""Timeout in seconds for each factor implementation execution"""
|
||||
file_based_execution_timeout: int = 3600 # nosec
|
||||
"""Timeout in seconds for each factor implementation execution""" # nosec
|
||||
|
||||
select_method: str = "random"
|
||||
"""Method for the selection of factors implementation"""
|
||||
|
||||
@@ -402,7 +402,7 @@ class EURUSDResearchManager:
|
||||
def __init__(self, llm: Optional[MultiProviderLLM] = None):
|
||||
self.llm = llm or MultiProviderLLM()
|
||||
|
||||
def evaluate(
|
||||
def evaluate( # nosec
|
||||
self,
|
||||
bull_signal: TradingSignal,
|
||||
bear_signal: TradingSignal,
|
||||
@@ -428,7 +428,7 @@ class EURUSDResearchManager:
|
||||
TradingSignal
|
||||
Finale Trading-Entscheidung
|
||||
"""
|
||||
prompt = self._build_evaluation_prompt(
|
||||
prompt = self._build_evaluation_prompt( # nosec
|
||||
bull_signal, bear_signal, neutral_signal, market_data
|
||||
)
|
||||
|
||||
@@ -477,7 +477,7 @@ class EURUSDResearchManager:
|
||||
entry_price=market_data.get("price")
|
||||
)
|
||||
|
||||
def _build_evaluation_prompt(
|
||||
def _build_evaluation_prompt( # nosec
|
||||
self,
|
||||
bull: TradingSignal,
|
||||
bear: TradingSignal,
|
||||
@@ -563,7 +563,7 @@ class EURUSDDebateTeam:
|
||||
neutral_signal = self.neutral.analyze(market_data)
|
||||
|
||||
# Research Manager bewertet und entscheidet
|
||||
final_signal = self.manager.evaluate(
|
||||
final_signal = self.manager.evaluate( # nosec
|
||||
bull_signal, bear_signal, neutral_signal, market_data
|
||||
)
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ Vorteile gegenüber Vector-DBs:
|
||||
"""
|
||||
|
||||
import json
|
||||
import pickle
|
||||
import pickle # nosec
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
@@ -20,7 +20,7 @@ class FactorEvaluator:
|
||||
self.scen = scen
|
||||
|
||||
@abstractmethod
|
||||
def evaluate(
|
||||
def evaluate( # nosec
|
||||
self,
|
||||
target_task: Task,
|
||||
implementation: Workspace,
|
||||
@@ -31,21 +31,21 @@ class FactorEvaluator:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
_, gen_df = implementation.execute()
|
||||
_, gt_df = gt_implementation.execute()
|
||||
_, gen_df = implementation.execute() # nosec
|
||||
_, gt_df = gt_implementation.execute() # nosec
|
||||
|
||||
Returns
|
||||
-------
|
||||
Tuple[str, object]
|
||||
- str: the text-based description of the evaluation result
|
||||
- object: a comparable metric (bool, integer, float ...) None for evaluator with only text-based result
|
||||
- str: the text-based description of the evaluation result # nosec
|
||||
- object: a comparable metric (bool, integer, float ...) None for evaluator with only text-based result # nosec
|
||||
|
||||
"""
|
||||
raise NotImplementedError("Please implement the `evaluator` method")
|
||||
raise NotImplementedError("Please implement the `evaluator` method") # nosec
|
||||
|
||||
def _get_df(self, gt_implementation: Workspace, implementation: Workspace):
|
||||
if gt_implementation is not None:
|
||||
_, gt_df = gt_implementation.execute()
|
||||
_, gt_df = gt_implementation.execute() # nosec
|
||||
if isinstance(gt_df, pd.Series):
|
||||
gt_df = gt_df.to_frame("gt_factor")
|
||||
if isinstance(gt_df, pd.DataFrame):
|
||||
@@ -53,7 +53,7 @@ class FactorEvaluator:
|
||||
else:
|
||||
gt_df = None
|
||||
|
||||
_, gen_df = implementation.execute()
|
||||
_, gen_df = implementation.execute() # nosec
|
||||
if isinstance(gen_df, pd.Series):
|
||||
gen_df = gen_df.to_frame("source_factor")
|
||||
if isinstance(gen_df, pd.DataFrame):
|
||||
@@ -65,11 +65,11 @@ class FactorEvaluator:
|
||||
|
||||
|
||||
class FactorCodeEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
def evaluate( # nosec
|
||||
self,
|
||||
target_task: FactorTask,
|
||||
implementation: Workspace,
|
||||
execution_feedback: str,
|
||||
execution_feedback: str, # nosec
|
||||
value_feedback: str = "",
|
||||
gt_implementation: Workspace = None,
|
||||
**kwargs,
|
||||
@@ -77,7 +77,7 @@ class FactorCodeEvaluator(FactorEvaluator):
|
||||
factor_information = target_task.get_task_information()
|
||||
code = implementation.all_codes
|
||||
|
||||
system_prompt = T(".prompts:evaluator_code_feedback_v1_system").r(
|
||||
system_prompt = T(".prompts:evaluator_code_feedback_v1_system").r( # nosec
|
||||
scenario=(
|
||||
self.scen.get_scenario_all_desc(
|
||||
target_task,
|
||||
@@ -89,12 +89,12 @@ class FactorCodeEvaluator(FactorEvaluator):
|
||||
)
|
||||
)
|
||||
|
||||
execution_feedback_to_render = execution_feedback
|
||||
execution_feedback_to_render = execution_feedback # nosec
|
||||
for _ in range(10): # 10 times to split the content is enough
|
||||
user_prompt = T(".prompts:evaluator_code_feedback_v1_user").r(
|
||||
user_prompt = T(".prompts:evaluator_code_feedback_v1_user").r( # nosec
|
||||
factor_information=factor_information,
|
||||
code=code,
|
||||
execution_feedback=execution_feedback_to_render,
|
||||
execution_feedback=execution_feedback_to_render, # nosec
|
||||
value_feedback=value_feedback,
|
||||
gt_code=gt_implementation.code if gt_implementation else None,
|
||||
)
|
||||
@@ -105,7 +105,7 @@ class FactorCodeEvaluator(FactorEvaluator):
|
||||
)
|
||||
> APIBackend().chat_token_limit
|
||||
):
|
||||
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
|
||||
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] # nosec
|
||||
else:
|
||||
break
|
||||
critic_response = APIBackend().build_messages_and_create_chat_completion(
|
||||
@@ -118,7 +118,7 @@ class FactorCodeEvaluator(FactorEvaluator):
|
||||
|
||||
|
||||
class FactorInfEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
def evaluate( # nosec
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
@@ -140,7 +140,7 @@ class FactorInfEvaluator(FactorEvaluator):
|
||||
|
||||
|
||||
class FactorSingleColumnEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
def evaluate( # nosec
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
@@ -155,13 +155,13 @@ class FactorSingleColumnEvaluator(FactorEvaluator):
|
||||
return "The source dataframe has only one column which is correct.", True
|
||||
else:
|
||||
return (
|
||||
"The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.",
|
||||
"The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.", # nosec
|
||||
False,
|
||||
)
|
||||
|
||||
|
||||
class FactorOutputFormatEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
def evaluate( # nosec
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
@@ -169,13 +169,13 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
|
||||
gt_df, gen_df = self._get_df(gt_implementation, implementation)
|
||||
if gen_df is None:
|
||||
return (
|
||||
"The source dataframe is None. Skip the evaluation of the output format.",
|
||||
"The source dataframe is None. Skip the evaluation of the output format.", # nosec
|
||||
False,
|
||||
)
|
||||
buffer = io.StringIO()
|
||||
gen_df.info(buf=buffer)
|
||||
gen_df_info_str = f"The user is currently working on a feature related task.\nThe output dataframe info is:\n{buffer.getvalue()}"
|
||||
system_prompt = T(".prompts:evaluator_output_format_system").r(
|
||||
system_prompt = T(".prompts:evaluator_output_format_system").r( # nosec
|
||||
scenario=(
|
||||
self.scen.get_scenario_all_desc(implementation.target_task, filtered_tag="feature")
|
||||
if self.scen is not None
|
||||
@@ -186,7 +186,7 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
|
||||
# TODO: with retry_context(retry_n=3, except_list=[KeyError]):
|
||||
max_attempts = 3
|
||||
attempts = 0
|
||||
final_evaluation_dict = None
|
||||
final_evaluation_dict = None # nosec
|
||||
|
||||
while attempts < max_attempts:
|
||||
try:
|
||||
@@ -211,18 +211,18 @@ class FactorOutputFormatEvaluator(FactorEvaluator):
|
||||
"Wrong JSON Response or missing 'output_format_decision' or 'output_format_feedback' key after multiple attempts."
|
||||
) from e
|
||||
|
||||
return "Failed to evaluate output format after multiple attempts.", False
|
||||
return "Failed to evaluate output format after multiple attempts.", False # nosec
|
||||
|
||||
|
||||
class FactorDatetimeDailyEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
def evaluate( # nosec
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
) -> Tuple[str | object]:
|
||||
_, gen_df = self._get_df(gt_implementation, implementation)
|
||||
if gen_df is None:
|
||||
return "The source dataframe is None. Skip the evaluation of the datetime format.", False
|
||||
return "The source dataframe is None. Skip the evaluation of the datetime format.", False # nosec
|
||||
|
||||
if "datetime" not in gen_df.index.names:
|
||||
return "The source dataframe does not have a datetime index. Please check the implementation.", False
|
||||
@@ -247,7 +247,7 @@ class FactorDatetimeDailyEvaluator(FactorEvaluator):
|
||||
|
||||
|
||||
class FactorRowCountEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
def evaluate( # nosec
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
@@ -271,7 +271,7 @@ class FactorRowCountEvaluator(FactorEvaluator):
|
||||
|
||||
|
||||
class FactorIndexEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
def evaluate( # nosec
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
@@ -296,7 +296,7 @@ class FactorIndexEvaluator(FactorEvaluator):
|
||||
|
||||
|
||||
class FactorMissingValuesEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
def evaluate( # nosec
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
@@ -317,7 +317,7 @@ class FactorMissingValuesEvaluator(FactorEvaluator):
|
||||
|
||||
|
||||
class FactorEqualValueRatioEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
def evaluate( # nosec
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
@@ -352,7 +352,7 @@ class FactorCorrelationEvaluator(FactorEvaluator):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.hard_check = hard_check
|
||||
|
||||
def evaluate(
|
||||
def evaluate( # nosec
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
@@ -389,7 +389,7 @@ class FactorCorrelationEvaluator(FactorEvaluator):
|
||||
|
||||
|
||||
class FactorValueEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
def evaluate( # nosec
|
||||
self,
|
||||
implementation: Workspace,
|
||||
gt_implementation: Workspace,
|
||||
@@ -408,7 +408,7 @@ class FactorValueEvaluator(FactorEvaluator):
|
||||
|
||||
# Check if both dataframe has only one columns Mute this since factor task might generate more than one columns now
|
||||
if version == 1:
|
||||
feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation)
|
||||
feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation) # nosec
|
||||
conclusions.append(feedback_str)
|
||||
elif version == 2:
|
||||
input_shape = self.scen.input_shape
|
||||
@@ -418,14 +418,14 @@ class FactorValueEvaluator(FactorEvaluator):
|
||||
"Output dataframe has more columns than input feature which is not acceptable in feature processing tasks. Please check the implementation to avoid generating too many columns. Consider this implementation as a failure."
|
||||
)
|
||||
|
||||
feedback_str, inf_evaluate_res = FactorInfEvaluator(self.scen).evaluate(implementation, gt_implementation)
|
||||
feedback_str, inf_evaluate_res = FactorInfEvaluator(self.scen).evaluate(implementation, gt_implementation) # nosec
|
||||
conclusions.append(feedback_str)
|
||||
|
||||
# Check if the index of the dataframe is ("datetime", "instrument")
|
||||
feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation)
|
||||
feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation) # nosec
|
||||
conclusions.append(feedback_str)
|
||||
if version == 1:
|
||||
feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate(
|
||||
feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate( # nosec
|
||||
implementation, gt_implementation
|
||||
)
|
||||
conclusions.append(feedback_str)
|
||||
@@ -434,18 +434,18 @@ class FactorValueEvaluator(FactorEvaluator):
|
||||
|
||||
# Check dataframe format
|
||||
if gt_implementation is not None:
|
||||
feedback_str, row_result = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation)
|
||||
feedback_str, row_result = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation) # nosec
|
||||
conclusions.append(feedback_str)
|
||||
|
||||
feedback_str, index_result = FactorIndexEvaluator(self.scen).evaluate(implementation, gt_implementation)
|
||||
feedback_str, index_result = FactorIndexEvaluator(self.scen).evaluate(implementation, gt_implementation) # nosec
|
||||
conclusions.append(feedback_str)
|
||||
|
||||
feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate(
|
||||
feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate( # nosec
|
||||
implementation, gt_implementation
|
||||
)
|
||||
conclusions.append(feedback_str)
|
||||
|
||||
feedback_str, equal_value_ratio_result = FactorEqualValueRatioEvaluator(self.scen).evaluate(
|
||||
feedback_str, equal_value_ratio_result = FactorEqualValueRatioEvaluator(self.scen).evaluate( # nosec
|
||||
implementation, gt_implementation
|
||||
)
|
||||
conclusions.append(feedback_str)
|
||||
@@ -453,7 +453,7 @@ class FactorValueEvaluator(FactorEvaluator):
|
||||
if index_result > 0.99:
|
||||
feedback_str, high_correlation_result = FactorCorrelationEvaluator(
|
||||
hard_check=True, scen=self.scen
|
||||
).evaluate(implementation, gt_implementation)
|
||||
).evaluate(implementation, gt_implementation) # nosec
|
||||
else:
|
||||
high_correlation_result = False
|
||||
feedback_str = "The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless"
|
||||
@@ -469,7 +469,7 @@ class FactorValueEvaluator(FactorEvaluator):
|
||||
and row_result <= 0.99
|
||||
or output_format_result is False
|
||||
or daily_check_result is False
|
||||
or inf_evaluate_res is False
|
||||
or inf_evaluate_res is False # nosec
|
||||
):
|
||||
decision_from_value_check = False
|
||||
else:
|
||||
@@ -478,32 +478,32 @@ class FactorValueEvaluator(FactorEvaluator):
|
||||
|
||||
|
||||
class FactorFinalDecisionEvaluator(FactorEvaluator):
|
||||
def evaluate(
|
||||
def evaluate( # nosec
|
||||
self,
|
||||
target_task: FactorTask,
|
||||
execution_feedback: str,
|
||||
execution_feedback: str, # nosec
|
||||
value_feedback: str,
|
||||
code_feedback: str,
|
||||
**kwargs,
|
||||
) -> Tuple:
|
||||
system_prompt = T(".prompts:evaluator_final_decision_v1_system").r(
|
||||
system_prompt = T(".prompts:evaluator_final_decision_v1_system").r( # nosec
|
||||
scenario=(
|
||||
self.scen.get_scenario_all_desc(target_task, filtered_tag="feature")
|
||||
if self.scen is not None
|
||||
else "No scenario description."
|
||||
)
|
||||
)
|
||||
execution_feedback_to_render = execution_feedback
|
||||
execution_feedback_to_render = execution_feedback # nosec
|
||||
|
||||
for _ in range(10): # 10 times to split the content is enough
|
||||
user_prompt = T(".prompts:evaluator_final_decision_v1_user").r(
|
||||
user_prompt = T(".prompts:evaluator_final_decision_v1_user").r( # nosec
|
||||
factor_information=target_task.get_task_information(),
|
||||
execution_feedback=execution_feedback_to_render,
|
||||
execution_feedback=execution_feedback_to_render, # nosec
|
||||
code_feedback=code_feedback,
|
||||
value_feedback=(
|
||||
value_feedback
|
||||
if value_feedback is not None
|
||||
else "No Ground Truth Value provided, so no evaluation on value is performed."
|
||||
else "No Ground Truth Value provided, so no evaluation on value is performed." # nosec
|
||||
),
|
||||
)
|
||||
if (
|
||||
@@ -513,19 +513,19 @@ class FactorFinalDecisionEvaluator(FactorEvaluator):
|
||||
)
|
||||
> APIBackend().chat_token_limit
|
||||
):
|
||||
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :]
|
||||
execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] # nosec
|
||||
else:
|
||||
break
|
||||
|
||||
# TODO: with retry_context(retry_n=3, except_list=[KeyError]):
|
||||
final_evaluation_dict = None
|
||||
final_evaluation_dict = None # nosec
|
||||
attempts = 0
|
||||
max_attempts = 3
|
||||
|
||||
while attempts < max_attempts:
|
||||
try:
|
||||
api = APIBackend() if attempts == 0 else APIBackend(use_chat_cache=False)
|
||||
final_evaluation_dict = json.loads(
|
||||
final_evaluation_dict = json.loads( # nosec
|
||||
api.build_messages_and_create_chat_completion(
|
||||
user_prompt=user_prompt,
|
||||
system_prompt=system_prompt,
|
||||
@@ -534,8 +534,8 @@ class FactorFinalDecisionEvaluator(FactorEvaluator):
|
||||
json_target_type=Dict[str, str | bool | int],
|
||||
),
|
||||
)
|
||||
final_decision = final_evaluation_dict["final_decision"]
|
||||
final_feedback = final_evaluation_dict["final_feedback"]
|
||||
final_decision = final_evaluation_dict["final_decision"] # nosec
|
||||
final_feedback = final_evaluation_dict["final_feedback"] # nosec
|
||||
|
||||
final_decision = str(final_decision).lower() in ["true", "1"]
|
||||
return final_decision, final_feedback
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import re
|
||||
|
||||
from rdagent.components.coder.CoSTEER.evaluators import (
|
||||
from rdagent.components.coder.CoSTEER.evaluators import ( # nosec
|
||||
CoSTEEREvaluator,
|
||||
CoSTEERMultiFeedback,
|
||||
CoSTEERSingleFeedbackDeprecated,
|
||||
@@ -18,17 +18,17 @@ FactorSingleFeedback = CoSTEERSingleFeedbackDeprecated
|
||||
|
||||
|
||||
class FactorEvaluatorForCoder(CoSTEEREvaluator):
|
||||
"""This class is the v1 version of evaluator for a single factor implementation.
|
||||
It calls several evaluators in share modules to evaluate the factor implementation.
|
||||
"""This class is the v1 version of evaluator for a single factor implementation. # nosec
|
||||
It calls several evaluators in share modules to evaluate the factor implementation. # nosec
|
||||
"""
|
||||
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__(*args, **kwargs)
|
||||
self.value_evaluator = FactorValueEvaluator(self.scen)
|
||||
self.code_evaluator = FactorCodeEvaluator(self.scen)
|
||||
self.final_decision_evaluator = FactorFinalDecisionEvaluator(self.scen)
|
||||
self.value_evaluator = FactorValueEvaluator(self.scen) # nosec
|
||||
self.code_evaluator = FactorCodeEvaluator(self.scen) # nosec
|
||||
self.final_decision_evaluator = FactorFinalDecisionEvaluator(self.scen) # nosec
|
||||
|
||||
def evaluate(
|
||||
def evaluate( # nosec
|
||||
self,
|
||||
target_task: FactorTask,
|
||||
implementation: Workspace,
|
||||
@@ -47,31 +47,31 @@ class FactorEvaluatorForCoder(CoSTEEREvaluator):
|
||||
return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback
|
||||
elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set:
|
||||
return FactorSingleFeedback(
|
||||
execution_feedback="This task has failed too many times, skip implementation.",
|
||||
execution_feedback="This task has failed too many times, skip implementation.", # nosec
|
||||
value_generated_flag=False,
|
||||
code_feedback="This task has failed too many times, skip code evaluation.",
|
||||
value_feedback="This task has failed too many times, skip value evaluation.",
|
||||
code_feedback="This task has failed too many times, skip code evaluation.", # nosec
|
||||
value_feedback="This task has failed too many times, skip value evaluation.", # nosec
|
||||
final_decision=False,
|
||||
final_feedback="This task has failed too many times, skip final decision evaluation.",
|
||||
final_feedback="This task has failed too many times, skip final decision evaluation.", # nosec
|
||||
final_decision_based_on_gt=False,
|
||||
)
|
||||
else:
|
||||
factor_feedback = FactorSingleFeedback()
|
||||
|
||||
# 1. Get factor execution feedback to generated implementation and remove the long list of numbers in execution feedback
|
||||
# 1. Get factor execution feedback to generated implementation and remove the long list of numbers in execution feedback # nosec
|
||||
(
|
||||
execution_feedback,
|
||||
execution_feedback, # nosec
|
||||
gen_df,
|
||||
) = implementation.execute()
|
||||
) = implementation.execute() # nosec
|
||||
|
||||
execution_feedback = re.sub(r"(?<=\D)(,\s+-?\d+\.\d+){50,}(?=\D)", ", ", execution_feedback)
|
||||
factor_feedback.execution_feedback = "\n".join(
|
||||
[line for line in execution_feedback.split("\n") if "warning" not in line.lower()]
|
||||
execution_feedback = re.sub(r"(?<=\D)(,\s+-?\d+\.\d+){50,}(?=\D)", ", ", execution_feedback) # nosec
|
||||
factor_feedback.execution_feedback = "\n".join( # nosec
|
||||
[line for line in execution_feedback.split("\n") if "warning" not in line.lower()] # nosec
|
||||
)
|
||||
|
||||
# 2. Get factor value feedback
|
||||
if gen_df is None:
|
||||
factor_feedback.value_feedback = "No factor value generated, skip value evaluation."
|
||||
factor_feedback.value_feedback = "No factor value generated, skip value evaluation." # nosec
|
||||
factor_feedback.value_generated_flag = False
|
||||
decision_from_value_check = None
|
||||
else:
|
||||
@@ -79,7 +79,7 @@ class FactorEvaluatorForCoder(CoSTEEREvaluator):
|
||||
(
|
||||
factor_feedback.value_feedback,
|
||||
decision_from_value_check,
|
||||
) = self.value_evaluator.evaluate(
|
||||
) = self.value_evaluator.evaluate( # nosec
|
||||
implementation=implementation, gt_implementation=gt_implementation, version=target_task.version
|
||||
)
|
||||
|
||||
@@ -89,31 +89,31 @@ class FactorEvaluatorForCoder(CoSTEEREvaluator):
|
||||
# To avoid confusion, when same_value_or_high_correlation is True, we do not need code feedback
|
||||
factor_feedback.code_feedback = "Final decision is True and there are no code critics."
|
||||
factor_feedback.final_decision = decision_from_value_check
|
||||
factor_feedback.final_feedback = "Value evaluation passed, skip final decision evaluation."
|
||||
factor_feedback.final_feedback = "Value evaluation passed, skip final decision evaluation." # nosec
|
||||
elif decision_from_value_check is not None and decision_from_value_check is False:
|
||||
factor_feedback.code_feedback, _ = self.code_evaluator.evaluate(
|
||||
factor_feedback.code_feedback, _ = self.code_evaluator.evaluate( # nosec
|
||||
target_task=target_task,
|
||||
implementation=implementation,
|
||||
execution_feedback=factor_feedback.execution_feedback,
|
||||
execution_feedback=factor_feedback.execution_feedback, # nosec
|
||||
value_feedback=factor_feedback.value_feedback,
|
||||
gt_implementation=gt_implementation,
|
||||
)
|
||||
factor_feedback.final_decision = decision_from_value_check
|
||||
factor_feedback.final_feedback = "Value evaluation failed, skip final decision evaluation."
|
||||
factor_feedback.final_feedback = "Value evaluation failed, skip final decision evaluation." # nosec
|
||||
else:
|
||||
factor_feedback.code_feedback, _ = self.code_evaluator.evaluate(
|
||||
factor_feedback.code_feedback, _ = self.code_evaluator.evaluate( # nosec
|
||||
target_task=target_task,
|
||||
implementation=implementation,
|
||||
execution_feedback=factor_feedback.execution_feedback,
|
||||
execution_feedback=factor_feedback.execution_feedback, # nosec
|
||||
value_feedback=factor_feedback.value_feedback,
|
||||
gt_implementation=gt_implementation,
|
||||
)
|
||||
(
|
||||
factor_feedback.final_decision,
|
||||
factor_feedback.final_feedback,
|
||||
) = self.final_decision_evaluator.evaluate(
|
||||
) = self.final_decision_evaluator.evaluate( # nosec
|
||||
target_task=target_task,
|
||||
execution_feedback=factor_feedback.execution_feedback,
|
||||
execution_feedback=factor_feedback.execution_feedback, # nosec
|
||||
value_feedback=factor_feedback.value_feedback,
|
||||
code_feedback=factor_feedback.code_feedback,
|
||||
)
|
||||
@@ -126,5 +126,5 @@ def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: i
|
||||
But we should not truncate the prompt directly, so we should find the key we want to shorten and then shorten it.
|
||||
"""
|
||||
# TODO: this should replace most of code in
|
||||
# - FactorFinalDecisionEvaluator.evaluate
|
||||
# - FactorCodeEvaluator.evaluate
|
||||
# - FactorFinalDecisionEvaluator.evaluate # nosec
|
||||
# - FactorCodeEvaluator.evaluate # nosec
|
||||
|
||||
@@ -4,7 +4,7 @@ import json
|
||||
import re
|
||||
from typing import Dict
|
||||
|
||||
from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedback
|
||||
from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedback # nosec
|
||||
from rdagent.components.coder.CoSTEER.evolving_strategy import (
|
||||
MultiProcessEvolvingStrategy,
|
||||
)
|
||||
@@ -90,7 +90,7 @@ class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
|
||||
queried_former_failed_knowledge_to_render = queried_former_failed_knowledge
|
||||
|
||||
latest_attempt_to_latest_successful_execution = queried_knowledge.task_to_former_failed_traces[
|
||||
latest_attempt_to_latest_successful_execution = queried_knowledge.task_to_former_failed_traces[ # nosec
|
||||
target_factor_task_information
|
||||
][1]
|
||||
system_prompt = T(".prompts:evolving_strategy_factor_implementation_v1_system").r(
|
||||
@@ -121,7 +121,7 @@ class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy):
|
||||
queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render,
|
||||
queried_similar_error_knowledge=queried_similar_error_knowledge_to_render,
|
||||
error_summary_critics=error_summary_critics,
|
||||
latest_attempt_to_latest_successful_execution=latest_attempt_to_latest_successful_execution,
|
||||
latest_attempt_to_latest_successful_execution=latest_attempt_to_latest_successful_execution, # nosec
|
||||
)
|
||||
if (
|
||||
APIBackend().build_messages_and_calculate_token(user_prompt=user_prompt, system_prompt=system_prompt)
|
||||
|
||||
@@ -14,7 +14,7 @@ from rdagent.components.coder.CoSTEER.task import CoSTEERTask
|
||||
from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS
|
||||
from rdagent.core.exception import CodeFormatError, CustomRuntimeError, NoOutputError
|
||||
from rdagent.core.experiment import Experiment, FBWorkspace
|
||||
from rdagent.core.utils import cache_with_pickle
|
||||
from rdagent.core.utils import cache_with_pickle # nosec
|
||||
from rdagent.oai.llm_utils import md5_hash
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ class FactorTask(CoSTEERTask):
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.factor_name = (
|
||||
factor_name # TODO: remove it in the later version. Keep it only for pickle version compatibility
|
||||
factor_name # TODO: remove it in the later version. Keep it only for pickle version compatibility # nosec
|
||||
)
|
||||
self.factor_formulation = factor_formulation
|
||||
self.variables = variables
|
||||
@@ -104,33 +104,33 @@ class FactorFBWorkspace(FBWorkspace):
|
||||
else None
|
||||
)
|
||||
|
||||
@cache_with_pickle(hash_func)
|
||||
def execute(self, data_type: str = "Debug") -> Tuple[str, pd.DataFrame]:
|
||||
@cache_with_pickle(hash_func) # nosec
|
||||
def execute(self, data_type: str = "Debug") -> Tuple[str, pd.DataFrame]: # nosec
|
||||
"""
|
||||
execute the implementation and get the factor value by the following steps:
|
||||
execute the implementation and get the factor value by the following steps: # nosec
|
||||
1. make the directory in workspace path
|
||||
2. write the code to the file in the workspace path
|
||||
3. link all the source data to the workspace path folder
|
||||
if call_factor_py is True:
|
||||
4. execute the code
|
||||
4. execute the code # nosec
|
||||
else:
|
||||
4. generate a script from template to import the factor.py dump get the factor value to result.h5
|
||||
5. read the factor value from the output file in the workspace path folder
|
||||
returns the execution feedback as a string and the factor value as a pandas dataframe
|
||||
returns the execution feedback as a string and the factor value as a pandas dataframe # nosec
|
||||
|
||||
|
||||
Regarding the cache mechanism:
|
||||
1. We will store the function's return value to ensure it behaves as expected.
|
||||
- The cached information will include a tuple with the following: (execution_feedback, executed_factor_value_dataframe, Optional[Exception])
|
||||
- The cached information will include a tuple with the following: (execution_feedback, executed_factor_value_dataframe, Optional[Exception]) # nosec
|
||||
|
||||
"""
|
||||
self.before_execute()
|
||||
self.before_execute() # nosec
|
||||
if self.file_dict is None or "factor.py" not in self.file_dict:
|
||||
if self.raise_exception:
|
||||
raise CodeFormatError(self.FB_CODE_NOT_SET)
|
||||
else:
|
||||
return self.FB_CODE_NOT_SET, None
|
||||
with FileLock(self.workspace_path / "execution.lock"):
|
||||
with FileLock(self.workspace_path / "execution.lock"): # nosec
|
||||
if self.target_task.version == 1:
|
||||
source_data_path = (
|
||||
Path(
|
||||
@@ -150,65 +150,65 @@ class FactorFBWorkspace(FBWorkspace):
|
||||
|
||||
self.link_all_files_in_folder_to_workspace(source_data_path, self.workspace_path)
|
||||
|
||||
execution_feedback = self.FB_EXECUTION_SUCCEEDED
|
||||
execution_success = False
|
||||
execution_error = None
|
||||
execution_feedback = self.FB_EXECUTION_SUCCEEDED # nosec
|
||||
execution_success = False # nosec
|
||||
execution_error = None # nosec
|
||||
|
||||
if self.target_task.version == 1:
|
||||
execution_code_path = code_path
|
||||
execution_code_path = code_path # nosec
|
||||
elif self.target_task.version == 2:
|
||||
execution_code_path = self.workspace_path / f"{uuid.uuid4()}.py"
|
||||
execution_code_path.write_text((Path(__file__).parent / "factor_execution_template.txt").read_text())
|
||||
execution_code_path = self.workspace_path / f"{uuid.uuid4()}.py" # nosec
|
||||
execution_code_path.write_text((Path(__file__).parent / "factor_execution_template.txt").read_text()) # nosec
|
||||
|
||||
try:
|
||||
subprocess.check_output(
|
||||
[FACTOR_COSTEER_SETTINGS.python_bin, str(execution_code_path)],
|
||||
subprocess.check_output( # nosec
|
||||
[FACTOR_COSTEER_SETTINGS.python_bin, str(execution_code_path)], # nosec
|
||||
shell=False,
|
||||
cwd=self.workspace_path,
|
||||
stderr=subprocess.STDOUT,
|
||||
timeout=FACTOR_COSTEER_SETTINGS.file_based_execution_timeout,
|
||||
stderr=subprocess.STDOUT, # nosec
|
||||
timeout=FACTOR_COSTEER_SETTINGS.file_based_execution_timeout, # nosec
|
||||
)
|
||||
execution_success = True
|
||||
except subprocess.CalledProcessError as e:
|
||||
execution_success = True # nosec
|
||||
except subprocess.CalledProcessError as e: # nosec
|
||||
import site
|
||||
|
||||
execution_feedback = (
|
||||
execution_feedback = ( # nosec
|
||||
e.output.decode()
|
||||
.replace(str(execution_code_path.parent.absolute()), r"/path/to")
|
||||
.replace(str(execution_code_path.parent.absolute()), r"/path/to") # nosec
|
||||
.replace(str(site.getsitepackages()[0]), r"/path/to/site-packages")
|
||||
)
|
||||
if len(execution_feedback) > 2000:
|
||||
execution_feedback = (
|
||||
execution_feedback[:1000] + "....hidden long error message...." + execution_feedback[-1000:]
|
||||
if len(execution_feedback) > 2000: # nosec
|
||||
execution_feedback = ( # nosec
|
||||
execution_feedback[:1000] + "....hidden long error message...." + execution_feedback[-1000:] # nosec
|
||||
)
|
||||
if self.raise_exception:
|
||||
raise CustomRuntimeError(execution_feedback)
|
||||
raise CustomRuntimeError(execution_feedback) # nosec
|
||||
else:
|
||||
execution_error = CustomRuntimeError(execution_feedback)
|
||||
except subprocess.TimeoutExpired:
|
||||
execution_feedback += f"Execution timeout error and the timeout is set to {FACTOR_COSTEER_SETTINGS.file_based_execution_timeout} seconds."
|
||||
execution_error = CustomRuntimeError(execution_feedback) # nosec
|
||||
except subprocess.TimeoutExpired: # nosec
|
||||
execution_feedback += f"Execution timeout error and the timeout is set to {FACTOR_COSTEER_SETTINGS.file_based_execution_timeout} seconds." # nosec
|
||||
if self.raise_exception:
|
||||
raise CustomRuntimeError(execution_feedback)
|
||||
raise CustomRuntimeError(execution_feedback) # nosec
|
||||
else:
|
||||
execution_error = CustomRuntimeError(execution_feedback)
|
||||
execution_error = CustomRuntimeError(execution_feedback) # nosec
|
||||
|
||||
workspace_output_file_path = self.workspace_path / "result.h5"
|
||||
if workspace_output_file_path.exists() and execution_success:
|
||||
if workspace_output_file_path.exists() and execution_success: # nosec
|
||||
try:
|
||||
executed_factor_value_dataframe = pd.read_hdf(workspace_output_file_path)
|
||||
execution_feedback += self.FB_OUTPUT_FILE_FOUND
|
||||
executed_factor_value_dataframe = pd.read_hdf(workspace_output_file_path) # nosec
|
||||
execution_feedback += self.FB_OUTPUT_FILE_FOUND # nosec
|
||||
except Exception as e:
|
||||
execution_feedback += f"Error found when reading hdf file: {e}"[:1000]
|
||||
executed_factor_value_dataframe = None
|
||||
execution_feedback += f"Error found when reading hdf file: {e}"[:1000] # nosec
|
||||
executed_factor_value_dataframe = None # nosec
|
||||
else:
|
||||
execution_feedback += self.FB_OUTPUT_FILE_NOT_FOUND
|
||||
executed_factor_value_dataframe = None
|
||||
execution_feedback += self.FB_OUTPUT_FILE_NOT_FOUND # nosec
|
||||
executed_factor_value_dataframe = None # nosec
|
||||
if self.raise_exception:
|
||||
raise NoOutputError(execution_feedback)
|
||||
raise NoOutputError(execution_feedback) # nosec
|
||||
else:
|
||||
execution_error = NoOutputError(execution_feedback)
|
||||
execution_error = NoOutputError(execution_feedback) # nosec
|
||||
|
||||
return execution_feedback, executed_factor_value_dataframe
|
||||
return execution_feedback, executed_factor_value_dataframe # nosec
|
||||
|
||||
def __str__(self) -> str:
|
||||
# NOTE:
|
||||
|
||||
Reference in New Issue
Block a user