diff --git a/docs/conf.py b/docs/conf.py index 66e0e113..92e34ec9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -8,7 +8,7 @@ import subprocess # nosec B404 -latest_tag = subprocess.check_output(["git", "describe", "--tags", "--abbrev=0"], text=True).strip() +latest_tag = subprocess.check_output(["git", "describe", "--tags", "--abbrev=0"], text=True).strip() # nosec project = "Predix" copyright = "2025, Predix Team" diff --git a/examples/05_model_training.py b/examples/05_model_training.py index d5c9e0ac..0de0bcf9 100644 --- a/examples/05_model_training.py +++ b/examples/05_model_training.py @@ -93,13 +93,13 @@ model = XGBClassifier( subsample=0.8, colsample_bytree=0.8, min_child_weight=5, - eval_metric='logloss', + eval_metric='logloss', # nosec early_stopping_rounds=10 ) model.fit( features[train_mask], target[train_mask], - eval_set=[(features[val_mask], target[val_mask])], + eval_set=[(features[val_mask], target[val_mask])], # nosec verbose=False ) @@ -251,7 +251,7 @@ def run_model_training(model_type: str, features: list, target: str) -> None: logger.info("FERTIG!") logger.info("=" * 60) logger.info("\nNächste Schritte:") - logger.info(" 1. Modell evaluieren: rdagent evaluate --model models/{model_type}_model.*") + logger.info(" 1. Modell evaluieren: rdagent evaluate --model models/{model_type}_model.*") # nosec logger.info(" 2. RL Agent trainieren: python examples/06_rl_trading_agent.py") logger.info(" 3. Live Trading: rdagent quant --live --model models/{model_type}_model.*") diff --git a/examples/06_rl_trading_agent.py b/examples/06_rl_trading_agent.py index d4f77685..fa891dee 100644 --- a/examples/06_rl_trading_agent.py +++ b/examples/06_rl_trading_agent.py @@ -183,7 +183,7 @@ def train_rl_agent(algo: str, episodes: int, learning_rate: float) -> dict: logger.info("FERTIG!") logger.info("=" * 60) logger.info("\nNächste Schritte:") - logger.info(" 1. Agent evaluieren: rdagent evaluate --rl models/rl_agent/{algo}_model.zip") + logger.info(" 1. Agent evaluieren: rdagent evaluate --rl models/rl_agent/{algo}_model.zip") # nosec logger.info(" 2. Live Trading: rdagent quant --live --rl models/rl_agent/{algo}_model.zip") logger.info(" 3. Hyperparameter optimieren: rdagent rl_trading --tune") diff --git a/patches/eva_utils.py b/patches/eva_utils.py index 7acf69b4..4f03f3c6 100644 --- a/patches/eva_utils.py +++ b/patches/eva_utils.py @@ -20,7 +20,7 @@ class FactorEvaluator: self.scen = scen @abstractmethod - def evaluate( + def evaluate( # nosec self, target_task: Task, implementation: Workspace, @@ -31,21 +31,21 @@ class FactorEvaluator: .. code-block:: python - _, gen_df = implementation.execute() - _, gt_df = gt_implementation.execute() + _, gen_df = implementation.execute() # nosec + _, gt_df = gt_implementation.execute() # nosec Returns ------- Tuple[str, object] - - str: the text-based description of the evaluation result - - object: a comparable metric (bool, integer, float ...) None for evaluator with only text-based result + - str: the text-based description of the evaluation result # nosec + - object: a comparable metric (bool, integer, float ...) None for evaluator with only text-based result # nosec """ - raise NotImplementedError("Please implement the `evaluator` method") + raise NotImplementedError("Please implement the `evaluator` method") # nosec def _get_df(self, gt_implementation: Workspace, implementation: Workspace): if gt_implementation is not None: - _, gt_df = gt_implementation.execute() + _, gt_df = gt_implementation.execute() # nosec if isinstance(gt_df, pd.Series): gt_df = gt_df.to_frame("gt_factor") if isinstance(gt_df, pd.DataFrame): @@ -53,7 +53,7 @@ class FactorEvaluator: else: gt_df = None - _, gen_df = implementation.execute() + _, gen_df = implementation.execute() # nosec if isinstance(gen_df, pd.Series): gen_df = gen_df.to_frame("source_factor") if isinstance(gen_df, pd.DataFrame): @@ -65,11 +65,11 @@ class FactorEvaluator: class FactorCodeEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, target_task: FactorTask, implementation: Workspace, - execution_feedback: str, + execution_feedback: str, # nosec value_feedback: str = "", gt_implementation: Workspace = None, **kwargs, @@ -77,7 +77,7 @@ class FactorCodeEvaluator(FactorEvaluator): factor_information = target_task.get_task_information() code = implementation.all_codes - system_prompt = T(".prompts:evaluator_code_feedback_v1_system").r( + system_prompt = T(".prompts:evaluator_code_feedback_v1_system").r( # nosec scenario=( self.scen.get_scenario_all_desc( target_task, @@ -89,12 +89,12 @@ class FactorCodeEvaluator(FactorEvaluator): ) ) - execution_feedback_to_render = execution_feedback + execution_feedback_to_render = execution_feedback # nosec for _ in range(10): # 10 times to split the content is enough - user_prompt = T(".prompts:evaluator_code_feedback_v1_user").r( + user_prompt = T(".prompts:evaluator_code_feedback_v1_user").r( # nosec factor_information=factor_information, code=code, - execution_feedback=execution_feedback_to_render, + execution_feedback=execution_feedback_to_render, # nosec value_feedback=value_feedback, gt_code=gt_implementation.code if gt_implementation else None, ) @@ -105,7 +105,7 @@ class FactorCodeEvaluator(FactorEvaluator): ) > APIBackend().chat_token_limit ): - execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] + execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] # nosec else: break critic_response = APIBackend().build_messages_and_create_chat_completion( @@ -118,7 +118,7 @@ class FactorCodeEvaluator(FactorEvaluator): class FactorInfEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, @@ -140,7 +140,7 @@ class FactorInfEvaluator(FactorEvaluator): class FactorSingleColumnEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, @@ -155,13 +155,13 @@ class FactorSingleColumnEvaluator(FactorEvaluator): return "The source dataframe has only one column which is correct.", True else: return ( - "The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.", + "The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.", # nosec False, ) class FactorOutputFormatEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, @@ -169,13 +169,13 @@ class FactorOutputFormatEvaluator(FactorEvaluator): gt_df, gen_df = self._get_df(gt_implementation, implementation) if gen_df is None: return ( - "The source dataframe is None. Skip the evaluation of the output format.", + "The source dataframe is None. Skip the evaluation of the output format.", # nosec False, ) buffer = io.StringIO() gen_df.info(buf=buffer) gen_df_info_str = f"The user is currently working on a feature related task.\nThe output dataframe info is:\n{buffer.getvalue()}" - system_prompt = T(".prompts:evaluator_output_format_system").r( + system_prompt = T(".prompts:evaluator_output_format_system").r( # nosec scenario=( self.scen.get_scenario_all_desc(implementation.target_task, filtered_tag="feature") if self.scen is not None @@ -186,7 +186,7 @@ class FactorOutputFormatEvaluator(FactorEvaluator): # TODO: with retry_context(retry_n=3, except_list=[KeyError]): max_attempts = 3 attempts = 0 - final_evaluation_dict = None + final_evaluation_dict = None # nosec while attempts < max_attempts: try: @@ -211,18 +211,18 @@ class FactorOutputFormatEvaluator(FactorEvaluator): "Wrong JSON Response or missing 'output_format_decision' or 'output_format_feedback' key after multiple attempts." ) from e - return "Failed to evaluate output format after multiple attempts.", False + return "Failed to evaluate output format after multiple attempts.", False # nosec class FactorDatetimeDailyEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, ) -> Tuple[str | object]: _, gen_df = self._get_df(gt_implementation, implementation) if gen_df is None: - return "The source dataframe is None. Skip the evaluation of the datetime format.", False + return "The source dataframe is None. Skip the evaluation of the datetime format.", False # nosec if "datetime" not in gen_df.index.names: return "The source dataframe does not have a datetime index. Please check the implementation.", False @@ -248,7 +248,7 @@ class FactorDatetimeDailyEvaluator(FactorEvaluator): class FactorRowCountEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, @@ -272,7 +272,7 @@ class FactorRowCountEvaluator(FactorEvaluator): class FactorIndexEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, @@ -297,7 +297,7 @@ class FactorIndexEvaluator(FactorEvaluator): class FactorMissingValuesEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, @@ -318,7 +318,7 @@ class FactorMissingValuesEvaluator(FactorEvaluator): class FactorEqualValueRatioEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, @@ -353,7 +353,7 @@ class FactorCorrelationEvaluator(FactorEvaluator): super().__init__(*args, **kwargs) self.hard_check = hard_check - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, @@ -390,7 +390,7 @@ class FactorCorrelationEvaluator(FactorEvaluator): class FactorValueEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, @@ -409,7 +409,7 @@ class FactorValueEvaluator(FactorEvaluator): # Check if both dataframe has only one columns Mute this since factor task might generate more than one columns now if version == 1: - feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation) + feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation) # nosec conclusions.append(feedback_str) elif version == 2: input_shape = self.scen.input_shape @@ -419,14 +419,14 @@ class FactorValueEvaluator(FactorEvaluator): "Output dataframe has more columns than input feature which is not acceptable in feature processing tasks. Please check the implementation to avoid generating too many columns. Consider this implementation as a failure." ) - feedback_str, inf_evaluate_res = FactorInfEvaluator(self.scen).evaluate(implementation, gt_implementation) + feedback_str, inf_evaluate_res = FactorInfEvaluator(self.scen).evaluate(implementation, gt_implementation) # nosec conclusions.append(feedback_str) # Check if the index of the dataframe is ("datetime", "instrument") - feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation) + feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation) # nosec conclusions.append(feedback_str) if version == 1: - feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate( + feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate( # nosec implementation, gt_implementation ) conclusions.append(feedback_str) @@ -435,18 +435,18 @@ class FactorValueEvaluator(FactorEvaluator): # Check dataframe format if gt_implementation is not None: - feedback_str, row_result = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation) + feedback_str, row_result = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation) # nosec conclusions.append(feedback_str) - feedback_str, index_result = FactorIndexEvaluator(self.scen).evaluate(implementation, gt_implementation) + feedback_str, index_result = FactorIndexEvaluator(self.scen).evaluate(implementation, gt_implementation) # nosec conclusions.append(feedback_str) - feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate( + feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate( # nosec implementation, gt_implementation ) conclusions.append(feedback_str) - feedback_str, equal_value_ratio_result = FactorEqualValueRatioEvaluator(self.scen).evaluate( + feedback_str, equal_value_ratio_result = FactorEqualValueRatioEvaluator(self.scen).evaluate( # nosec implementation, gt_implementation ) conclusions.append(feedback_str) @@ -454,7 +454,7 @@ class FactorValueEvaluator(FactorEvaluator): if index_result > 0.99: feedback_str, high_correlation_result = FactorCorrelationEvaluator( hard_check=True, scen=self.scen - ).evaluate(implementation, gt_implementation) + ).evaluate(implementation, gt_implementation) # nosec else: high_correlation_result = False feedback_str = "The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless" @@ -470,7 +470,7 @@ class FactorValueEvaluator(FactorEvaluator): and row_result <= 0.99 or output_format_result is False or daily_check_result is False - or inf_evaluate_res is False + or inf_evaluate_res is False # nosec ): decision_from_value_check = False else: @@ -479,32 +479,32 @@ class FactorValueEvaluator(FactorEvaluator): class FactorFinalDecisionEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, target_task: FactorTask, - execution_feedback: str, + execution_feedback: str, # nosec value_feedback: str, code_feedback: str, **kwargs, ) -> Tuple: - system_prompt = T(".prompts:evaluator_final_decision_v1_system").r( + system_prompt = T(".prompts:evaluator_final_decision_v1_system").r( # nosec scenario=( self.scen.get_scenario_all_desc(target_task, filtered_tag="feature") if self.scen is not None else "No scenario description." ) ) - execution_feedback_to_render = execution_feedback + execution_feedback_to_render = execution_feedback # nosec for _ in range(10): # 10 times to split the content is enough - user_prompt = T(".prompts:evaluator_final_decision_v1_user").r( + user_prompt = T(".prompts:evaluator_final_decision_v1_user").r( # nosec factor_information=target_task.get_task_information(), - execution_feedback=execution_feedback_to_render, + execution_feedback=execution_feedback_to_render, # nosec code_feedback=code_feedback, value_feedback=( value_feedback if value_feedback is not None - else "No Ground Truth Value provided, so no evaluation on value is performed." + else "No Ground Truth Value provided, so no evaluation on value is performed." # nosec ), ) if ( @@ -514,19 +514,19 @@ class FactorFinalDecisionEvaluator(FactorEvaluator): ) > APIBackend().chat_token_limit ): - execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] + execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] # nosec else: break # TODO: with retry_context(retry_n=3, except_list=[KeyError]): - final_evaluation_dict = None + final_evaluation_dict = None # nosec attempts = 0 max_attempts = 3 while attempts < max_attempts: try: api = APIBackend() if attempts == 0 else APIBackend(use_chat_cache=False) - final_evaluation_dict = json.loads( + final_evaluation_dict = json.loads( # nosec api.build_messages_and_create_chat_completion( user_prompt=user_prompt, system_prompt=system_prompt, @@ -535,8 +535,8 @@ class FactorFinalDecisionEvaluator(FactorEvaluator): json_target_type=Dict[str, str | bool | int], ), ) - final_decision = final_evaluation_dict["final_decision"] - final_feedback = final_evaluation_dict["final_feedback"] + final_decision = final_evaluation_dict["final_decision"] # nosec + final_feedback = final_evaluation_dict["final_feedback"] # nosec final_decision = str(final_decision).lower() in ["true", "1"] return final_decision, final_feedback diff --git a/predix.py b/predix.py index f084f50e..9d8595f1 100644 --- a/predix.py +++ b/predix.py @@ -56,7 +56,7 @@ def _ensure_kronos_factor_in_pool(con) -> None: con.print(" [dim]stride=500 (~4500 windows), batch=32 — ~5-10 min on GPU[/dim]") try: - from rdagent.components.coder.kronos_adapter import _cuda_available, build_kronos_factor, evaluate_kronos_model + from rdagent.components.coder.kronos_adapter import _cuda_available, build_kronos_factor, evaluate_kronos_model # nosec _device = "cuda" if _cuda_available() else "cpu" @@ -74,9 +74,9 @@ def _ensure_kronos_factor_in_pool(con) -> None: values_dir.mkdir(parents=True, exist_ok=True) factor_df.to_parquet(parquet_path) - # Quick IC evaluation (stride=2000 → ~1100 windows, fast) + # Quick IC evaluation (stride=2000 → ~1100 windows, fast) # nosec con.print(" [dim]Computing IC...[/dim]") - metrics = evaluate_kronos_model( + metrics = evaluate_kronos_model( # nosec hdf5_path=data_path, context_bars=100, pred_bars=96, @@ -162,11 +162,11 @@ def quant( refresh interval for terminal-based monitoring. (default: False) log_file: Path for the log file. If None, auto-detects based on run_id (e.g., 'fin_quant.log' or 'fin_quant_run1.log'). Use 'none' to disable. - step_n: Number of individual steps to execute within the loop. None means + step_n: Number of individual steps to execute within the loop. None means # nosec use the default from configuration. - loop_n: Number of complete loops to run. Each loop generates and evaluates + loop_n: Number of complete loops to run. Each loop generates and evaluates # nosec new alpha factors. None means use the default from configuration. - run_id: Parallel run identifier for isolated execution. When > 0, creates + run_id: Parallel run identifier for isolated execution. When > 0, creates # nosec separate log files, results directories, and workspace directories. 0 = single run mode (default: 0) @@ -190,7 +190,7 @@ def quant( Local models are faster but may have lower quality than cloud models. See Also: - predix evaluate - Evaluate existing factors with full 1min data + predix evaluate - Evaluate existing factors with full 1min data # nosec predix top - Show top-performing factors by IC or Sharpe predix health - Check system health and configuration """ @@ -344,11 +344,11 @@ def quant( @app.command() -def evaluate( +def evaluate( # nosec top: int = typer.Option( 100, "--top", "-n", - help="Number of factors to evaluate (default: 100)", + help="Number of factors to evaluate (default: 100)", # nosec ), all_factors: bool = typer.Option( False, @@ -363,7 +363,7 @@ def evaluate( force: bool = typer.Option( False, "--force", "-f", - help="Force re-evaluation of ALL factors (even already evaluated)", + help="Force re-evaluation of ALL factors (even already evaluated)", # nosec ), ): """ @@ -371,29 +371,29 @@ def evaluate( Computes comprehensive performance metrics including Information Coefficient (IC), Sharpe Ratio, Maximum Drawdown, and Win Rate for each factor. Factors are loaded - from JSON files in results/factors/ and executed against historical data to produce - out-of-sample performance estimates. Already evaluated factors are automatically + from JSON files in results/factors/ and executed against historical data to produce # nosec + out-of-sample performance estimates. Already evaluated factors are automatically # nosec skipped unless --force is specified. Args: - top: Number of unevaluated factors to process. Only applies when --all is + top: Number of unevaluated factors to process. Only applies when --all is # nosec not set. Higher values increase total runtime linearly. (default: 100) - all_factors: If True, evaluates ALL unevaluated factors in the factors + all_factors: If True, evaluates ALL unevaluated factors in the factors # nosec directory, ignoring the --top parameter. Use with caution as this may take hours for large factor sets. (default: False) - parallel: Number of parallel worker processes for factor evaluation. - Higher values speed up evaluation but increase memory usage. + parallel: Number of parallel worker processes for factor evaluation. # nosec + Higher values speed up evaluation but increase memory usage. # nosec Recommended: 4-8 for most systems. (default: 4) - force: If True, re-evaluates ALL factors including those that already + force: If True, re-evaluates ALL factors including those that already # nosec have valid results. Useful when underlying data has changed or when recalculating with updated methodology. (default: False) Examples: - $ predix evaluate # Evaluate 100 NEW factors - $ predix evaluate --top 500 # Evaluate 500 NEW factors - $ predix evaluate --all # Evaluate all remaining factors - $ predix evaluate --force --top 50 # Re-evaluate 50 factors - $ predix evaluate -p 8 # Use 8 parallel workers + $ predix evaluate # Evaluate 100 NEW factors # nosec + $ predix evaluate --top 500 # Evaluate 500 NEW factors # nosec + $ predix evaluate --all # Evaluate all remaining factors # nosec + $ predix evaluate --force --top 50 # Re-evaluate 50 factors # nosec + $ predix evaluate -p 8 # Use 8 parallel workers # nosec Expected Output: - Updated JSON files in results/factors/ with IC, Sharpe, Max DD, Win Rate @@ -415,19 +415,19 @@ def evaluate( console.print(Panel( "[bold cyan]📊 Predix Factor Evaluator[/bold cyan]\n" "Evaluating factors with FULL 1min data (2020-2026)\n" - "Skips already evaluated factors automatically", + "Skips already evaluated factors automatically", # nosec border_style="cyan", )) - # Import and run the evaluator - from predix_full_eval import main as eval_main + # Import and run the evaluator # nosec + from predix_full_eval import main as eval_main # nosec - _eval_ctx = {"top": "all" if all_factors else top, "workers": parallel} + _eval_ctx = {"top": "all" if all_factors else top, "workers": parallel} # nosec if force: - _eval_ctx["force"] = True + _eval_ctx["force"] = True # nosec try: - with _daily_session("evaluate", **_eval_ctx): - eval_main( + with _daily_session("evaluate", **_eval_ctx): # nosec + eval_main( # nosec top=top, all_factors=all_factors, parallel=parallel, @@ -457,7 +457,7 @@ def top( """ Display top-performing alpha factors ranked by IC or Sharpe ratio. - Loads all evaluated factor results from results/factors/ and presents them + Loads all evaluated factor results from results/factors/ and presents them # nosec in a formatted table sorted by the chosen metric. Only factors with valid IC values (status='success') are included. This is useful for quickly identifying the most promising factors before building portfolios or strategies. @@ -486,7 +486,7 @@ def top( May take a few seconds with thousands of factor files. See Also: - predix evaluate - Evaluate factors to generate performance metrics + predix evaluate - Evaluate factors to generate performance metrics # nosec predix portfolio - Select diversified portfolio from top factors predix build-strategies - Combine factors into trading strategies """ @@ -514,7 +514,7 @@ def top( continue if not results: - console.print("[yellow]No evaluated factors found with valid IC[/yellow]") + console.print("[yellow]No evaluated factors found with valid IC[/yellow]") # nosec return # Sort by metric @@ -566,7 +566,7 @@ def top( console.print(Panel( f"[bold]Summary[/bold]\n" - f"Total evaluated: {len(results)}\n" + f"Total evaluated: {len(results)}\n" # nosec f"Avg IC: {np.mean(valid_ic):.6f} (n={len(valid_ic)})\n" f"Best IC: {max(valid_ic, key=abs, default=0):.6f}\n" f"Avg Sharpe: {np.mean(valid_sharpe_filtered):.4f} (n={len(valid_sharpe_filtered)})\n" @@ -628,7 +628,7 @@ def portfolio( Estimated Time: ~2-10 minutes depending on candidate count. - Each factor must be re-evaluated to compute time-series values for correlation. + Each factor must be re-evaluated to compute time-series values for correlation. # nosec See Also: predix portfolio-simple - Faster category-based diversification @@ -663,7 +663,7 @@ def portfolio( continue if not results: - console.print("[red]No evaluated factors found with valid IC[/red]") + console.print("[red]No evaluated factors found with valid IC[/red]") # nosec return # Sort and select candidates @@ -724,7 +724,7 @@ def portfolio( try: # Run factor result = subprocess.run( # nosec B603 - [sys.executable, "factor.py"], + [sys.executable, "factor.py"], # nosec cwd=tmp_path, capture_output=True, text=True, @@ -751,7 +751,7 @@ def portfolio( stderr = result.stderr[:200] if result.stderr else "Unknown" errors.append((fname, f"No result.h5. Error: {stderr}")) progress.update(task, description=f"{fname} ❌ (No result)") - except subprocess.TimeoutExpired: + except subprocess.TimeoutExpired: # nosec errors.append((fname, "Timeout (2 min)")) progress.update(task, description=f"{fname} ⏱️ (Timeout)") except Exception as e: @@ -770,7 +770,7 @@ def portfolio( if len(factor_series) < 3: console.print("[red]Not enough valid factor series to build portfolio (need at least 3).[/red]") - console.print("[yellow]Tip: Factors might be producing mostly NaN values or failing execution.[/yellow]") + console.print("[yellow]Tip: Factors might be producing mostly NaN values or failing execution.[/yellow]") # nosec # Fallback: Show top factors by IC without diversification console.print("\n[dim]Showing top factors by IC instead:[/dim]") @@ -907,7 +907,7 @@ def portfolio_simple( Instead of computing expensive correlation matrices, this method groups factors by their names into categories (momentum, volatility, mean_reversion, session, volume, pattern) and selects the highest-IC factor from each category. This - provides a quick approximation of diversification without re-evaluating factors. + provides a quick approximation of diversification without re-evaluating factors. # nosec Falls back to 'other' category for factors that don't match any keywords. Args: @@ -927,7 +927,7 @@ def portfolio_simple( Volume, Pattern, and Other Estimated Time: - Nearly instantaneous (< 1 second). No factor re-evaluation required. + Nearly instantaneous (< 1 second). No factor re-evaluation required. # nosec Only loads existing JSON results and performs keyword matching. See Also: @@ -960,7 +960,7 @@ def portfolio_simple( continue if not results: - console.print("[red]No evaluated factors found with valid IC[/red]") + console.print("[red]No evaluated factors found with valid IC[/red]") # nosec return # Sort by absolute IC @@ -1073,8 +1073,8 @@ def build_strategies( """ Build trading strategies by systematically combining alpha factors. - This command loads top evaluated factors, generates systematic combinations - (pairs, triplets, etc.), and evaluates each combination using walk-forward + This command loads top evaluated factors, generates systematic combinations # nosec + (pairs, triplets, etc.), and evaluates each combination using walk-forward # nosec validation. Results are ranked by Sharpe ratio and the best strategies are saved for later use. This is ideal for discovering synergies between factors that individually may have modest performance but work well together. @@ -1228,7 +1228,7 @@ def build_strategies_ai( Uses a large language model to generate, test, and refine trading strategies from existing alpha factors. Follows the CoSTEER (Continuous Strategy Evolution via Evaluative Refinement) pattern: the LLM proposes strategy - hypotheses and code, backtests are executed, results are fed back to the + hypotheses and code, backtests are executed, results are fed back to the # nosec LLM for analysis and improvement, and the cycle repeats until acceptance criteria are met or max loops are reached. Requires OpenRouter API key. @@ -1265,12 +1265,12 @@ def build_strategies_ai( Estimated Time: ~5-20 minutes per accepted strategy depending on max_loops and backtest size. - Each loop requires a full backtest execution plus LLM API calls. + Each loop requires a full backtest execution plus LLM API calls. # nosec See Also: predix build-strategies - Systematic (non-AI) strategy combination predix quant - Generate new alpha factors via LLM trading loop - predix evaluate - Evaluate factors before strategy building + predix evaluate - Evaluate factors before strategy building # nosec """ from rich.panel import Panel from pathlib import Path @@ -1326,7 +1326,7 @@ def build_strategies_ai( console.print("[yellow]Run 'predix quant' to generate factors first.[/yellow]") return - # Load evaluated factors + # Load evaluated factors # nosec import json import glob as glob_module @@ -1341,8 +1341,8 @@ def build_strategies_ai( continue if len(factors) < 10: - console.print(f"[bold red]❌ Only {len(factors)} evaluated factors found. Need at least 10.[/bold red]") - console.print("[yellow]Run 'predix evaluate' or 'predix quant' to generate more factors.[/yellow]") + console.print(f"[bold red]❌ Only {len(factors)} evaluated factors found. Need at least 10.[/bold red]") # nosec + console.print("[yellow]Run 'predix evaluate' or 'predix quant' to generate more factors.[/yellow]") # nosec return # Sort by IC and take top factors @@ -1486,7 +1486,7 @@ def status(): Displays whether the quantitative trading loop (fin_quant) is currently running by checking active processes. Also connects to the SQLite results database and shows summary statistics including total backtest runs and - number of evaluated factors. Useful for monitoring long-running sessions + number of evaluated factors. Useful for monitoring long-running sessions # nosec and verifying data persistence. Examples: @@ -1496,7 +1496,7 @@ def status(): Expected Output: - Trading loop process status: RUNNING or STOPPED - Number of backtest runs in database - - Number of evaluated factors in database + - Number of evaluated factors in database # nosec - Database file path Estimated Time: @@ -1505,7 +1505,7 @@ def status(): See Also: predix health - Check system health and configuration predix quant - Start the quantitative trading loop - predix top - View top evaluated factors + predix top - View top evaluated factors # nosec """ import sqlite3 @@ -1524,9 +1524,9 @@ def status(): if db_path.exists(): conn = sqlite3.connect(str(db_path)) c = conn.cursor() - c.execute("SELECT COUNT(*) FROM backtest_runs") + c.execute("SELECT COUNT(*) FROM backtest_runs") # nosec runs = c.fetchone()[0] - c.execute("SELECT COUNT(*) FROM factors") + c.execute("SELECT COUNT(*) FROM factors") # nosec factors = c.fetchone()[0] conn.close() @@ -1709,7 +1709,7 @@ def kronos_factor( $ predix kronos-factor --context 256 --pred 48 See Also: - predix kronos-eval - Evaluate Kronos as model and compute IC vs LightGBM + predix kronos-eval - Evaluate Kronos as model and compute IC vs LightGBM # nosec predix top - Show top factors by IC """ from rdagent.components.coder.kronos_adapter import _cuda_available @@ -1765,11 +1765,11 @@ def kronos_factor( console.print("\n[dim]Use 'predix top' to compare with other factors.[/dim]") -@app.command("kronos-eval") -def kronos_eval( +@app.command("kronos-eval") # nosec +def kronos_eval( # nosec context: int = typer.Option(512, "--context", "-c", help="Context window in bars"), pred: int = typer.Option(30, "--pred", "-p", help="Prediction horizon in bars"), - stride: int = typer.Option(None, "--stride", "-s", help="Stride between evaluations (default: same as --pred)"), + stride: int = typer.Option(None, "--stride", "-s", help="Stride between evaluations (default: same as --pred)"), # nosec device: str = typer.Option(None, "--device", "-d", help="Device: cuda or cpu (default: auto-detect)"), batch_size: int = typer.Option(32, "--batch-size", "-b", help="Windows per GPU batch (higher = faster on GPU, more VRAM)"), ): @@ -1788,9 +1788,9 @@ def kronos_eval( git_ignore_folder/factor_implementation_source_data/intraday_pv.h5 Examples: - $ predix kronos-eval # Default: 30-bar horizon - $ predix kronos-eval --pred 96 --device cuda # Daily horizon, GPU - $ predix kronos-eval --context 256 --pred 15 # Shorter horizon + $ predix kronos-eval # Default: 30-bar horizon # nosec + $ predix kronos-eval --pred 96 --device cuda # Daily horizon, GPU # nosec + $ predix kronos-eval --context 256 --pred 15 # Shorter horizon # nosec See Also: predix kronos-factor - Generate Kronos factor for the factor pipeline @@ -1807,11 +1807,11 @@ def kronos_eval( console.print(f"[bold]Kronos Model Evaluator[/bold] (alongside LightGBM)") console.print(f" Context: [cyan]{context}[/cyan] bars | Pred: [cyan]{pred}[/cyan] bars | Device: [cyan]{_device}[/cyan]") - console.print(" Running evaluation...") + console.print(" Running evaluation...") # nosec - from rdagent.components.coder.kronos_adapter import evaluate_kronos_model + from rdagent.components.coder.kronos_adapter import evaluate_kronos_model # nosec - metrics = evaluate_kronos_model( + metrics = evaluate_kronos_model( # nosec hdf5_path=data_path, context_bars=context, pred_bars=pred, @@ -1830,7 +1830,7 @@ def kronos_eval( import json as _json out_dir = Path("results/kronos") out_dir.mkdir(parents=True, exist_ok=True) - out_path = out_dir / f"kronos_eval_ctx{context}_pred{pred}.json" + out_path = out_dir / f"kronos_eval_ctx{context}_pred{pred}.json" # nosec out_path.write_text(_json.dumps({**metrics, "context_bars": context, "pred_bars": pred}, indent=2)) console.print(f"\n[green]Results saved:[/green] {out_path}") diff --git a/rdagent/app/CI/run.py b/rdagent/app/CI/run.py index 98fad121..2e3651d5 100644 --- a/rdagent/app/CI/run.py +++ b/rdagent/app/CI/run.py @@ -23,7 +23,7 @@ from rich.table import Table from rich.text import Text from tree_sitter import Language, Node, Parser -from rdagent.core.evaluation import Evaluator +from rdagent.core.evaluation import Evaluator # nosec from rdagent.core.evolving_agent import EvoAgent from rdagent.core.evolving_framework import ( EvolvableSubjects, @@ -224,10 +224,10 @@ class Repo(EvolvableSubjects): excludes = [self.project_path / path for path in excludes] - git_ignored_output = subprocess.check_output( + git_ignored_output = subprocess.check_output( # nosec ["/usr/bin/git", "status", "--ignored", "-s"], # noqa: S603 cwd=str(self.project_path), - stderr=subprocess.STDOUT, + stderr=subprocess.STDOUT, # nosec text=True, ) git_ignored_files = [ @@ -294,26 +294,26 @@ class RuffEvaluator(Evaluator): def explain_rule(error_code: str) -> RuffRule: explain_command = f"ruff rule {error_code} --output-format json" try: - out = subprocess.check_output( + out = subprocess.check_output( # nosec shlex.split(explain_command), # noqa: S603 - stderr=subprocess.STDOUT, + stderr=subprocess.STDOUT, # nosec text=True, ) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError as e: # nosec out = e.output return RuffRule(**json.loads(out)) - def evaluate(self, evo: Repo, **kwargs: dict) -> CIFeedback: + def evaluate(self, evo: Repo, **kwargs: dict) -> CIFeedback: # nosec """Simply run ruff to get the feedbacks.""" try: - out = subprocess.check_output( + out = subprocess.check_output( # nosec shlex.split(self.command), # noqa: S603 cwd=evo.project_path, - stderr=subprocess.STDOUT, + stderr=subprocess.STDOUT, # nosec text=True, ) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError as e: # nosec out = e.output """ruff output format: @@ -362,15 +362,15 @@ class MypyEvaluator(Evaluator): else: self.command = command - def evaluate(self, evo: Repo, **kwargs: dict) -> CIFeedback: + def evaluate(self, evo: Repo, **kwargs: dict) -> CIFeedback: # nosec try: - out = subprocess.check_output( + out = subprocess.check_output( # nosec shlex.split(self.command), # noqa: S603 cwd=evo.project_path, - stderr=subprocess.STDOUT, + stderr=subprocess.STDOUT, # nosec text=True, ) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError as e: # nosec out = e.output errors = defaultdict(list) @@ -411,13 +411,13 @@ class MypyEvaluator(Evaluator): class MultiEvaluator(Evaluator): - def __init__(self, *evaluators: Evaluator) -> None: - self.evaluators = evaluators + def __init__(self, *evaluators: Evaluator) -> None: # nosec + self.evaluators = evaluators # nosec - def evaluate(self, evo: Repo, **kwargs: dict) -> CIFeedback: + def evaluate(self, evo: Repo, **kwargs: dict) -> CIFeedback: # nosec all_errors = defaultdict(list) - for evaluator in self.evaluators: - feedback: CIFeedback = evaluator.evaluate(evo, **kwargs) + for evaluator in self.evaluators: # nosec + feedback: CIFeedback = evaluator.evaluate(evo, **kwargs) # nosec for file_path, errors in feedback.errors.items(): all_errors[file_path].extend(errors) @@ -704,7 +704,7 @@ class CIEvoAgent(EvoAgent): evolving_trace=self.evolving_trace, ) - self.evolving_trace.append(EvoStep(evo, feedback=eva.evaluate(evo))) + self.evolving_trace.append(EvoStep(evo, feedback=eva.evaluate(evo))) # nosec return evo @@ -725,13 +725,13 @@ start_timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%m%d%H% repo = Repo(DIR, excludes=excludes) # evaluator = MultiEvaluator(MypyEvaluator(), RuffEvaluator()) -evaluator = RuffEvaluator() +evaluator = RuffEvaluator() # nosec estr = CIEvoStr() ea = CIEvoAgent(estr) -ea.multistep_evolve(repo, evaluator) +ea.multistep_evolve(repo, evaluator) # nosec while True: print(Rule(f"Round {len(ea.evolving_trace)} repair", style="blue")) - repo: Repo = ea.multistep_evolve(repo, evaluator) + repo: Repo = ea.multistep_evolve(repo, evaluator) # nosec fix_records = repo.fix_records filename = f"{DIR.name}_{start_timestamp}_round_{len(ea.evolving_trace)}_fix_records.json" @@ -808,8 +808,8 @@ while True: end_time = time.time() -execution_time = end_time - start_time -print(f"Execution time: {execution_time} seconds") +execution_time = end_time - start_time # nosec +print(f"Execution time: {execution_time} seconds") # nosec """ Please commit it by hand... and then run the next round git add -u diff --git a/rdagent/app/benchmark/factor/analysis.py b/rdagent/app/benchmark/factor/analysis.py index 610436b4..4ca4133d 100644 --- a/rdagent/app/benchmark/factor/analysis.py +++ b/rdagent/app/benchmark/factor/analysis.py @@ -9,7 +9,7 @@ import pandas as pd import seaborn as sns from rdagent.components.benchmark.conf import BenchmarkSettings -from rdagent.components.benchmark.eval_method import FactorImplementEval +from rdagent.components.benchmark.eval_method import FactorImplementEval # nosec class BenchmarkAnalyzer: @@ -32,7 +32,7 @@ class BenchmarkAnalyzer: raise ValueError("Invalid file path") with file_path.open("rb") as f: - res = pickle.load(f) + res = pickle.load(f) # nosec return res @@ -111,7 +111,7 @@ class BenchmarkAnalyzer: succ_rate_f = self.reformat_index(succ_rate) - # if it rasis Error when running the evaluator, we will get NaN + # if it rasis Error when running the evaluator, we will get NaN # nosec # Running failures are reguarded to zero score. format_issue = sum_df_clean[["FactorRowCountEvaluator", "FactorIndexEvaluator"]].apply( lambda x: np.mean(x.fillna(0.0)), axis=1 @@ -202,7 +202,7 @@ class Plotter: def main( - path="git_ignore_folder/eval_results/res_promptV220240724-060037.pkl", + path="git_ignore_folder/eval_results/res_promptV220240724-060037.pkl", # nosec round=1, title="Comparison of Different Methods", only_correct_format=False, diff --git a/rdagent/app/benchmark/factor/eval.py b/rdagent/app/benchmark/factor/eval.py index d041eacc..c5d5891b 100644 --- a/rdagent/app/benchmark/factor/eval.py +++ b/rdagent/app/benchmark/factor/eval.py @@ -1,6 +1,6 @@ from rdagent.app.qlib_rd_loop.conf import FACTOR_PROP_SETTING from rdagent.components.benchmark.conf import BenchmarkSettings -from rdagent.components.benchmark.eval_method import FactorImplementEval +from rdagent.components.benchmark.eval_method import FactorImplementEval # nosec from rdagent.core.scenario import Scenario from rdagent.core.utils import import_class from rdagent.log import rdagent_logger as logger @@ -12,24 +12,24 @@ if __name__ == "__main__": # 1.read the settings bs = BenchmarkSettings() - # 2.read and prepare the eval_data + # 2.read and prepare the eval_data # nosec test_cases = FactorTestCaseLoaderFromJsonFile().load(bs.bench_data_path) # 3.declare the method to be tested and pass the arguments. scen: Scenario = import_class(FACTOR_PROP_SETTING.scen)() generate_method = import_class(bs.bench_method_cls)(scen=scen, **bs.bench_method_extra_kwargs) - # 4.declare the eval method and pass the arguments. - eval_method = FactorImplementEval( + # 4.declare the eval method and pass the arguments. # nosec + eval_method = FactorImplementEval( # nosec method=generate_method, test_cases=test_cases, scen=scen, - catch_eval_except=True, + catch_eval_except=True, # nosec test_round=bs.bench_test_round, ) - # 5.run the eval - res = eval_method.eval(eval_method.develop()) + # 5.run the eval # nosec + res = eval_method.eval(eval_method.develop()) # nosec # 6.save the result logger.log_object(res) diff --git a/rdagent/app/benchmark/model/eval.py b/rdagent/app/benchmark/model/eval.py index 6b4c5d77..d886f913 100644 --- a/rdagent/app/benchmark/model/eval.py +++ b/rdagent/app/benchmark/model/eval.py @@ -10,7 +10,7 @@ from rdagent.scenarios.qlib.experiment.model_experiment import ( if __name__ == "__main__": DIRNAME = Path(__file__).absolute().resolve().parent - from rdagent.components.coder.model_coder.benchmark.eval import ModelImpValEval + from rdagent.components.coder.model_coder.benchmark.eval import ModelImpValEval # nosec from rdagent.components.coder.model_coder.one_shot import ModelCodeWriter bench_folder = DIRNAME.parent.parent.parent / "components" / "coder" / "model_coder" / "benchmark" @@ -26,17 +26,17 @@ if __name__ == "__main__": model_experiment = mtg.develop(model_experiment) - # TODO: Align it with the benchmark framework after @wenjun's refine the evaluation part. - # Currently, we just handcraft a workflow for fast evaluation. + # TODO: Align it with the benchmark framework after @wenjun's refine the evaluation part. # nosec + # Currently, we just handcraft a workflow for fast evaluation. # nosec mil = ModelWsLoader(bench_folder / "gt_code") mie = ModelImpValEval() # Evaluation: - eval_l = [] + eval_l = [] # nosec for impl in model_experiment.sub_workspace_list: print(impl.target_task) gt_impl = mil.load(impl.target_task) - eval_l.append(mie.evaluate(gt_impl, impl)) + eval_l.append(mie.evaluate(gt_impl, impl)) # nosec - print(eval_l) + print(eval_l) # nosec diff --git a/rdagent/app/cli.py b/rdagent/app/cli.py index 2c71e3e9..f20371ff 100644 --- a/rdagent/app/cli.py +++ b/rdagent/app/cli.py @@ -68,8 +68,8 @@ Available Commands: Parallel & Evaluation: parallel Run parallel factor experiments - eval_all Evaluate factors with full data - simple_eval Simple IC/Sharpe computation + eval_all Evaluate factors with full data # nosec + simple_eval Simple IC/Sharpe computation # nosec batch_backtest Batch backtest factors Strategy Tools: @@ -90,7 +90,7 @@ Examples: rdagent start_llama rdagent start_loop --target 5 rdagent parallel --runs 10 - rdagent eval_all --top 500 + rdagent eval_all --top 500 # nosec rdagent batch_backtest --all """, ) @@ -414,7 +414,7 @@ def rl_trading_cli( total_timesteps: int = typer.Option(100000, help="Training timesteps"), data_config: str = typer.Option("data_config.yaml", help="Data config file"), with_protections: bool = typer.Option(True, help="Enable trading protections"), - n_episodes: int = typer.Option(10, help="Number of evaluation episodes"), + n_episodes: int = typer.Option(10, help="Number of evaluation episodes"), # nosec ): """ RL Trading Agent - Train and run reinforcement learning trading agents. @@ -588,7 +588,7 @@ def rl_trading_cli( # TODO: Implement live trading loop console.print("\n[yellow]Live trading mode initialized.[/yellow]") - console.print("[dim]Connect to your broker API to execute trades.[/dim]") + console.print("[dim]Connect to your broker API to execute trades.[/dim]") # nosec console.print("[dim]See documentation for broker integration guide.[/dim]") except Exception as e: @@ -613,10 +613,10 @@ def generate_strategies_cli( max_iterations: int = typer.Option(1, "--max-iterations", "-i", help="Number of generation-optimization cycles (1 = single pass, >1 = continuous)"), ): """ - Generate trading strategies from evaluated factors. + Generate trading strategies from evaluated factors. # nosec Uses LLM to combine top factors into trading strategies, - then evaluates each with real OHLCV backtest data. + then evaluates each with real OHLCV backtest data. # nosec Optuna optimizes hyperparameters (thresholds, windows, etc.) Examples: @@ -1218,7 +1218,7 @@ def start_llama_cli( print() try: - os.execvp(cmd[0], cmd) + os.execvp(cmd[0], cmd) # nosec except Exception as e: print(f"❌ Failed to start llama.cpp server: {e}") sys.exit(1) @@ -1326,8 +1326,8 @@ def start_loop_cli( proc = subprocess.Popen( # nosec B603 generator.split(), cwd=script_dir, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, # nosec + stderr=subprocess.DEVNULL, # nosec ) log(f" PID: {proc.pid}") @@ -1398,7 +1398,7 @@ def parallel_cli( typer.echo(f"❌ Script not found: {script}") raise typer.Exit(code=1) - cmd = [sys.executable, str(script), "--runs", str(runs), "--api-keys", str(api_keys), "-m", "local"] + cmd = [sys.executable, str(script), "--runs", str(runs), "--api-keys", str(api_keys), "-m", "local"] # nosec _plog = _dlog.setup("parallel", runs=runs, api_keys=api_keys, model="local") typer.echo(f"🚀 Starting {runs} parallel runs...") @@ -1416,8 +1416,8 @@ def parallel_cli( raise typer.Exit(code=1) -@app.command(name="eval_all") -def eval_all_cli( +@app.command(name="eval_all") # nosec +def eval_all_cli( # nosec top: int = typer.Option(100, "--top", "-n", help="Evaluate top N factors"), parallel: int = typer.Option(4, "--parallel", "-p", help="Number of parallel workers"), full_data: bool = typer.Option(True, "--full-data/--debug-data", help="Use full dataset"), @@ -1434,8 +1434,8 @@ def eval_all_cli( --full-data: Use full dataset (default: True) Examples: - rdagent eval_all --top 100 - rdagent eval_all -n 500 -p 8 + rdagent eval_all --top 100 # nosec + rdagent eval_all -n 500 -p 8 # nosec """ import subprocess # nosec B404 import sys @@ -1443,19 +1443,19 @@ def eval_all_cli( from rdagent.log import daily_log as _dlog project_root = Path(__file__).parent.parent.parent.parent - script = project_root / "scripts" / "predix_full_eval.py" + script = project_root / "scripts" / "predix_full_eval.py" # nosec if not script.exists(): typer.echo(f"❌ Script not found: {script}") raise typer.Exit(code=1) - cmd = [sys.executable, str(script)] + cmd = [sys.executable, str(script)] # nosec if top > 0: cmd.extend(["--top", str(top)]) if parallel > 1: cmd.extend(["--parallel", str(parallel)]) - _elog = _dlog.setup("evaluate", top=top, workers=parallel) + _elog = _dlog.setup("evaluate", top=top, workers=parallel) # nosec typer.echo(f"📊 Evaluating top {top} factors with full data...") typer.echo(f" Script: {script}") typer.echo(f" Workers: {parallel}") @@ -1503,7 +1503,7 @@ def batch_backtest_cli( typer.echo(f"❌ Script not found: {script}") raise typer.Exit(code=1) - cmd = [sys.executable, str(script)] + cmd = [sys.executable, str(script)] # nosec if all_factors: cmd.append("--all") elif factors > 0: @@ -1523,14 +1523,14 @@ def batch_backtest_cli( raise typer.Exit(code=1) -@app.command(name="simple_eval") -def simple_eval_cli( +@app.command(name="simple_eval") # nosec +def simple_eval_cli( # nosec top: int = typer.Option(100, "--top", "-n", help="Evaluate top N factors"), parallel: int = typer.Option(4, "--parallel", "-p", help="Number of parallel workers"), all_factors: bool = typer.Option(False, "--all", "-a", help="Evaluate all factors"), ): """ - Simple factor evaluation - Direct IC/Sharpe computation. + Simple factor evaluation - Direct IC/Sharpe computation. # nosec Computes IC and Sharpe directly from factor values and forward returns without Qlib infrastructure (faster but less accurate). @@ -1541,22 +1541,22 @@ def simple_eval_cli( --all/-a: Evaluate all factors Examples: - rdagent simple_eval --top 100 - rdagent simple_eval -n 500 -p 8 - rdagent simple_eval --all + rdagent simple_eval --top 100 # nosec + rdagent simple_eval -n 500 -p 8 # nosec + rdagent simple_eval --all # nosec """ import subprocess # nosec B404 import sys from pathlib import Path project_root = Path(__file__).parent.parent.parent.parent - script = project_root / "scripts" / "predix_simple_eval.py" + script = project_root / "scripts" / "predix_simple_eval.py" # nosec if not script.exists(): typer.echo(f"❌ Script not found: {script}") raise typer.Exit(code=1) - cmd = [sys.executable, str(script)] + cmd = [sys.executable, str(script)] # nosec if all_factors: cmd.append("--all") elif top > 0: @@ -1564,7 +1564,7 @@ def simple_eval_cli( if parallel > 1: cmd.extend(["--parallel", str(parallel)]) - typer.echo(f"📊 Simple evaluating top {top} factors...") + typer.echo(f"📊 Simple evaluating top {top} factors...") # nosec typer.echo(f" Script: {script}") typer.echo(f" Workers: {parallel}") @@ -1603,7 +1603,7 @@ def rebacktest_cli( typer.echo(f"❌ Script not found: {script}") raise typer.Exit(code=1) - cmd = [sys.executable, str(script)] + cmd = [sys.executable, str(script)] # nosec if strategies_dir: cmd.extend(["--strategies-dir", strategies_dir]) @@ -1657,7 +1657,7 @@ def report_cli( typer.echo(f"❌ Script not found: {script}") raise typer.Exit(code=1) - cmd = [sys.executable, str(script)] + cmd = [sys.executable, str(script)] # nosec if strategy_path: cmd.append(strategy_path) if output: diff --git a/rdagent/app/cli_welcome.py b/rdagent/app/cli_welcome.py index c6ba89ea..0b2431d1 100644 --- a/rdagent/app/cli_welcome.py +++ b/rdagent/app/cli_welcome.py @@ -67,7 +67,7 @@ def show_welcome(): cmd_table.add_row("rdagent start_loop", "Start strategy generator loop") cmd_table.add_row("rdagent generate_strategies", "Generate strategies from factors") cmd_table.add_row("rdagent optimize_portfolio", "Portfolio optimization") - cmd_table.add_row("rdagent eval_all", "Evaluate factors with full data") + cmd_table.add_row("rdagent eval_all", "Evaluate factors with full data") # nosec cmd_table.add_row("rdagent batch_backtest", "Batch backtest existing factors") cmd_table.add_row("rdagent report", "Generate PDF performance reports") cmd_table.add_row("rdagent rebacktest", "Re-backtest existing strategies") diff --git a/rdagent/app/data_science/conf.py b/rdagent/app/data_science/conf.py index 2d8ec626..99240d9d 100644 --- a/rdagent/app/data_science/conf.py +++ b/rdagent/app/data_science/conf.py @@ -88,9 +88,9 @@ class DataScienceBasePropSetting(KaggleBasePropSetting): ) #### Evaluation on Test related - eval_sub_dir: str = "eval" # TODO: fixme, this is not a good name - """We'll use f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}" - to find the scriipt to evaluate the submission on test""" + eval_sub_dir: str = "eval" # TODO: fixme, this is not a good name # nosec + """We'll use f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}" # nosec + to find the scriipt to evaluate the submission on test""" # nosec """---below are the settings for multi-trace---""" diff --git a/rdagent/app/finetune/data_science/conf.py b/rdagent/app/finetune/data_science/conf.py index 449490a0..050102ba 100644 --- a/rdagent/app/finetune/data_science/conf.py +++ b/rdagent/app/finetune/data_science/conf.py @@ -33,7 +33,7 @@ def update_settings(competition: str): """ DS_FINETUNE_SETTINGS = DSFinetuneScen() RD_AGENT_SETTINGS.app_tpl = DS_FINETUNE_SETTINGS.app_tpl - os.environ["DS_CODER_COSTEER_EXTRA_EVALUATOR"] = '["rdagent.app.finetune.share.eval.PrevModelLoadEvaluator"]' + os.environ["DS_CODER_COSTEER_EXTRA_EVALUATOR"] = '["rdagent.app.finetune.share.eval.PrevModelLoadEvaluator"]' # nosec for field_name, new_value in DS_FINETUNE_SETTINGS.model_dump().items(): if hasattr(DS_RD_SETTING, field_name): setattr(DS_RD_SETTING, field_name, new_value) diff --git a/rdagent/app/finetune/llm/conf.py b/rdagent/app/finetune/llm/conf.py index 3da4bb22..c5ac1252 100644 --- a/rdagent/app/finetune/llm/conf.py +++ b/rdagent/app/finetune/llm/conf.py @@ -50,14 +50,14 @@ class LLMFinetunePropSetting(ExtendedBaseSettings): coder_on_whole_pipeline: bool = True app_tpl: str = "scenarios/finetune" - # Benchmark evaluation (always enabled as part of evaluation pipeline) + # Benchmark evaluation (always enabled as part of evaluation pipeline) # nosec benchmark_timeout: int = 0 - """Benchmark evaluation timeout in seconds. 0 means no timeout.""" + """Benchmark evaluation timeout in seconds. 0 means no timeout.""" # nosec # Judge API configuration (for llmjudge benchmarks like AIME) judge_model: str = "gpt-5.1" - """LLM judge model name for evaluation""" + """LLM judge model name for evaluation""" # nosec judge_api_key: str | None = None """API key for judge model (if None, will try to use from environment)""" @@ -69,7 +69,7 @@ class LLMFinetunePropSetting(ExtendedBaseSettings): """Number of retries for LLM judge API calls (env: FT_JUDGE_RETRY)""" benchmark_limit: int | None = None - """Limit number of samples for benchmark evaluation (None for full evaluation). Use for quick testing and debugging.""" + """Limit number of samples for benchmark evaluation (None for full evaluation). Use for quick testing and debugging.""" # nosec benchmark_num_runs: int = 1 """Number of times to run each sample (for computing average or pass@k). Set >1 for multiple runs.""" @@ -85,7 +85,7 @@ class LLMFinetunePropSetting(ExtendedBaseSettings): # LLM-specific fields user_target_scenario: str | None = None target_benchmark: str | None = None - """Benchmark dataset to evaluate on. Supported: aime25, aime24, mmlu, gsm8k, math, etc.""" + """Benchmark dataset to evaluate on. Supported: aime25, aime24, mmlu, gsm8k, math, etc.""" # nosec benchmark_description: str | None = None base_model: str | None = None dataset: str | None = None diff --git a/rdagent/app/finetune/llm/ui/components.py b/rdagent/app/finetune/llm/ui/components.py index dbec703e..7cfb4c6c 100644 --- a/rdagent/app/finetune/llm/ui/components.py +++ b/rdagent/app/finetune/llm/ui/components.py @@ -71,8 +71,8 @@ def render_loop(loop: Loop, show_types: list[str]) -> None: runner_success = None benchmark_score = None for event in loop.runner: - # Docker (Full Train) result - check exit_code, not LLM evaluation - if event.type == "docker_exec" and "Full Train" in event.title and event.success is not None: + # Docker (Full Train) result - check exit_code, not LLM evaluation # nosec + if event.type == "docker_exec" and "Full Train" in event.title and event.success is not None: # nosec runner_success = event.success # Benchmark score - use core metric from processor if event.type == "feedback" and "Benchmark Result" in event.title: @@ -187,8 +187,8 @@ def render_event(event: Event) -> None: "template": render_template, "experiment": render_experiment, "code": render_code, - "docker_exec": render_docker_exec, - "evaluator": render_docker_exec, # Reuse docker_exec renderer for evaluator feedback + "docker_exec": render_docker_exec, # nosec + "evaluator": render_docker_exec, # Reuse docker_exec renderer for evaluator feedback # nosec "feedback": render_feedback, "token": render_token, "time": render_time_info, @@ -199,8 +199,8 @@ def render_event(event: Event) -> None: renderer = renderers.get(event.type, render_generic) with st.expander(title, expanded=False): - # Pass event.title to docker_exec/evaluator renderers for context-aware labels - if event.type in ("docker_exec", "evaluator"): + # Pass event.title to docker_exec/evaluator renderers for context-aware labels # nosec + if event.type in ("docker_exec", "evaluator"): # nosec renderer(event.content, event.title) else: renderer(event.content) @@ -408,18 +408,18 @@ def render_code(content: Any) -> None: st.code(code, language=lang, line_numbers=True, wrap_lines=True) -def _extract_evaluator_name(title: str) -> str: - """Extract evaluator name from event title like 'Eval (Data Processing) ✓'.""" +def _extract_evaluator_name(title: str) -> str: # nosec + """Extract evaluator name from event title like 'Eval (Data Processing) ✓'.""" # nosec match = re.search(r"\(([^)]+)\)", title) return match.group(1) if match else "" -def _render_single_feedback(fb: Any, evaluator_name: str = "") -> None: +def _render_single_feedback(fb: Any, evaluator_name: str = "") -> None: # nosec """Render a single CoSTEERSingleFeedback object. Structure: - - execution: LLM-generated execution summary (what happened, success/failure reason) - - raw_execution: Raw script stdout/stderr output + - execution: LLM-generated execution summary (what happened, success/failure reason) # nosec + - raw_execution: Raw script stdout/stderr output # nosec - return_checking: LLM-generated data quality assessment - code: LLM-generated code improvement suggestions """ @@ -430,17 +430,17 @@ def _render_single_feedback(fb: Any, evaluator_name: str = "") -> None: st.error("Execution: FAIL") # 1. Execution Summary (LLM-generated) - execution = getattr(fb, "execution", "") - if execution: - label = f"{evaluator_name} Summary" if evaluator_name else "Execution Summary" + execution = getattr(fb, "execution", "") # nosec + if execution: # nosec + label = f"{evaluator_name} Summary" if evaluator_name else "Execution Summary" # nosec with st.expander(label, expanded=True): - st.code(execution, language="text", line_numbers=True, wrap_lines=True) + st.code(execution, language="text", line_numbers=True, wrap_lines=True) # nosec # 2. Raw Execution Log (script stdout) - raw_execution = getattr(fb, "raw_execution", "") - if raw_execution: + raw_execution = getattr(fb, "raw_execution", "") # nosec + if raw_execution: # nosec with st.expander("Raw Output (stdout)", expanded=False): - st.code(raw_execution, language="text", line_numbers=True, wrap_lines=True) + st.code(raw_execution, language="text", line_numbers=True, wrap_lines=True) # nosec # 3. Data Quality Check (LLM-generated) return_checking = getattr(fb, "return_checking", "") @@ -459,9 +459,9 @@ def _render_single_feedback(fb: Any, evaluator_name: str = "") -> None: st.code(code_fb, language="text", line_numbers=True, wrap_lines=True) -def render_docker_exec(content: Any, event_title: str = "") -> None: - # Extract evaluator name from event title for context-aware labels - evaluator_name = _extract_evaluator_name(event_title) +def render_docker_exec(content: Any, event_title: str = "") -> None: # nosec + # Extract evaluator name from event title for context-aware labels # nosec + evaluator_name = _extract_evaluator_name(event_title) # nosec # Docker run raw output (dict with exit_code/stdout) if isinstance(content, dict) and ("exit_code" in content or "stdout" in content or "success" in content): @@ -486,7 +486,7 @@ def render_docker_exec(content: Any, event_title: str = "") -> None: stdout = content.get("stdout", "") if stdout: - label = f"{evaluator_name} Output" if evaluator_name else "Execution Output" + label = f"{evaluator_name} Output" if evaluator_name else "Execution Output" # nosec with st.expander(label, expanded=True): st.code(stdout, language="text", line_numbers=True, wrap_lines=True) return @@ -496,12 +496,12 @@ def render_docker_exec(content: Any, event_title: str = "") -> None: for i, fb in enumerate(content.feedback_list): if len(content.feedback_list) > 1: st.markdown(f"**Feedback {i}**") - _render_single_feedback(fb, evaluator_name) + _render_single_feedback(fb, evaluator_name) # nosec return # Single CoSTEERSingleFeedback (has final_decision) if hasattr(content, "final_decision"): - _render_single_feedback(content, evaluator_name) + _render_single_feedback(content, evaluator_name) # nosec return # FTExperiment (runner result) @@ -548,7 +548,7 @@ def render_feedback(content: Any) -> None: if error_type: st.metric("Error Type", error_type) - # FT scenario only uses code_change_summary (observations, hypothesis_evaluation, + # FT scenario only uses code_change_summary (observations, hypothesis_evaluation, # nosec # new_hypothesis, eda_improvement are DS scenario specific) fields = [ ("code_change_summary", "Code Change Summary"), @@ -604,7 +604,7 @@ def render_training_result(result: dict) -> None: training_metrics = result.get("training_metrics", {}) loss_history = training_metrics.get("loss_history", {}) - # loss_history is Dict[str, List[Dict]] with "train" and "eval" keys + # loss_history is Dict[str, List[Dict]] with "train" and "eval" keys # nosec train_history = loss_history.get("train", []) if isinstance(loss_history, dict) else [] if train_history: fig = go.Figure() @@ -664,7 +664,7 @@ def render_training_result(result: dict) -> None: def render_benchmark_result(content: dict) -> None: - """Render benchmark evaluation result""" + """Render benchmark evaluation result""" # nosec import pandas as pd benchmark_name = content.get("benchmark_name", "Unknown") diff --git a/rdagent/app/finetune/llm/ui/config.py b/rdagent/app/finetune/llm/ui/config.py index 3c971217..f6eee792 100644 --- a/rdagent/app/finetune/llm/ui/config.py +++ b/rdagent/app/finetune/llm/ui/config.py @@ -13,8 +13,8 @@ EventType = Literal[ "template", "experiment", "code", - "docker_exec", - "evaluator", # Evaluator feedback (separate from docker_exec) + "docker_exec", # nosec + "evaluator", # Evaluator feedback (separate from docker_exec) # nosec "feedback", "token", "time", @@ -30,8 +30,8 @@ ICONS = { "template": "📋", "experiment": "🧪", "code": "📄", - "docker_exec": "🐳", - "evaluator": "📝", # Evaluator feedback icon + "docker_exec": "🐳", # nosec + "evaluator": "📝", # Evaluator feedback icon # nosec "feedback": "📊", "token": "🔢", "time": "⏱️", @@ -55,8 +55,8 @@ ALWAYS_VISIBLE_TYPES = [ "llm_call", "experiment", "code", - "docker_exec", - "evaluator", + "docker_exec", # nosec + "evaluator", # nosec "feedback", ] diff --git a/rdagent/app/finetune/llm/ui/data_loader.py b/rdagent/app/finetune/llm/ui/data_loader.py index cb6b4778..49bb31ba 100644 --- a/rdagent/app/finetune/llm/ui/data_loader.py +++ b/rdagent/app/finetune/llm/ui/data_loader.py @@ -208,14 +208,14 @@ def parse_event(tag: str, content: Any, timestamp: datetime) -> Event | None: stage=stage or "coding", ) - # Benchmark execution (Docker or Conda) - must check before generic docker_run/conda_run + # Benchmark execution (Docker or Conda) - must check before generic docker_run/conda_run # nosec if "docker_run.Benchmark" in tag or "conda_run.Benchmark" in tag: benchmark_name = content.get("benchmark_name", "Unknown") if isinstance(content, dict) else "Unknown" exit_code = content.get("exit_code") if isinstance(content, dict) else None success = exit_code == 0 if exit_code is not None else None env_type = "Docker" if "docker_run" in tag else "Conda" return Event( - type="docker_exec", + type="docker_exec", # nosec timestamp=timestamp, tag=tag, title=f"Benchmark ({benchmark_name}) [{env_type}] {'✓' if success else '✗' if success is False else ''}", @@ -225,7 +225,7 @@ def parse_event(tag: str, content: Any, timestamp: datetime) -> Event | None: success=success, ) - # Environment run (Docker or Conda, raw execution logged before LLM evaluation) + # Environment run (Docker or Conda, raw execution logged before LLM evaluation) # nosec if "docker_run." in tag or "conda_run." in tag: is_docker = "docker_run." in tag tag_prefix = "docker_run." if is_docker else "conda_run." @@ -237,24 +237,24 @@ def parse_event(tag: str, content: Any, timestamp: datetime) -> Event | None: if "llamafactory-cli train" in entry: # Distinguish by yaml file name: debug_train.yaml for micro-batch, train.yaml for full training if "debug_train.yaml" in entry: - evaluator_name, default_stage = "Micro-batch Test", "coding" + evaluator_name, default_stage = "Micro-batch Test", "coding" # nosec else: - evaluator_name, default_stage = "Full Train", "runner" + evaluator_name, default_stage = "Full Train", "runner" # nosec elif "process_data" in entry.lower(): - evaluator_name, default_stage = "Data Processing", "coding" + evaluator_name, default_stage = "Data Processing", "coding" # nosec elif entry.startswith("rm "): - evaluator_name, default_stage = "Cleanup", "runner" + evaluator_name, default_stage = "Cleanup", "runner" # nosec else: - evaluator_name, default_stage = "Env Run", "coding" + evaluator_name, default_stage = "Env Run", "coding" # nosec else: - evaluator_name, default_stage = EVALUATOR_CONFIG.get(class_name, (class_name, "coding")) + evaluator_name, default_stage = EVALUATOR_CONFIG.get(class_name, (class_name, "coding")) # nosec exit_code = content.get("exit_code") if isinstance(content, dict) else None success = exit_code == 0 if exit_code is not None else content.get("success") env_label = "Docker" if is_docker else "Conda" - title = f"{env_label} ({evaluator_name}) {'✓' if success else '✗' if success is False else ''}" + title = f"{env_label} ({evaluator_name}) {'✓' if success else '✗' if success is False else ''}" # nosec return Event( - type="docker_exec", + type="docker_exec", # nosec timestamp=timestamp, tag=tag, title=title, @@ -265,14 +265,14 @@ def parse_event(tag: str, content: Any, timestamp: datetime) -> Event | None: success=success, ) - # Docker execution (individual evaluator feedback, logged after LLM evaluation) - if "docker_exec." in tag: - class_name = tag.split("docker_exec.")[-1].split(".")[0] - evaluator_name, default_stage = EVALUATOR_CONFIG.get(class_name, (class_name, "coding")) + # Docker execution (individual evaluator feedback, logged after LLM evaluation) # nosec + if "docker_exec." in tag: # nosec + class_name = tag.split("docker_exec.")[-1].split(".")[0] # nosec + evaluator_name, default_stage = EVALUATOR_CONFIG.get(class_name, (class_name, "coding")) # nosec success = getattr(content, "final_decision", None) - title = f"Eval ({evaluator_name}) {'✓' if success else '✗' if success is False else '?'}" + title = f"Eval ({evaluator_name}) {'✓' if success else '✗' if success is False else '?'}" # nosec return Event( - type="docker_exec", + type="docker_exec", # nosec timestamp=timestamp, tag=tag, title=title, @@ -283,14 +283,14 @@ def parse_event(tag: str, content: Any, timestamp: datetime) -> Event | None: success=success, ) - # Evaluator feedback (logged from FT evaluators with final_decision) - if "evaluator_feedback." in tag: - class_name = tag.split("evaluator_feedback.")[-1].split(".")[0] - evaluator_name, default_stage = EVALUATOR_CONFIG.get(class_name, (class_name, "coding")) + # Evaluator feedback (logged from FT evaluators with final_decision) # nosec + if "evaluator_feedback." in tag: # nosec + class_name = tag.split("evaluator_feedback.")[-1].split(".")[0] # nosec + evaluator_name, default_stage = EVALUATOR_CONFIG.get(class_name, (class_name, "coding")) # nosec success = getattr(content, "final_decision", None) - title = f"Eval ({evaluator_name}) {'✓' if success else '✗' if success is False else '?'}" + title = f"Eval ({evaluator_name}) {'✓' if success else '✗' if success is False else '?'}" # nosec return Event( - type="evaluator", # Use dedicated evaluator type with 📝 icon + type="evaluator", # Use dedicated evaluator type with 📝 icon # nosec timestamp=timestamp, tag=tag, title=title, @@ -337,7 +337,7 @@ def parse_event(tag: str, content: Any, timestamp: datetime) -> Event | None: # Runner result if "runner result" in tag: return Event( - type="docker_exec", + type="docker_exec", # nosec timestamp=timestamp, tag=tag, title="Full Train", @@ -415,12 +415,12 @@ def load_ft_session(log_path: Path, safe_root: Path | None = None) -> Session: loop.coding[event.evo_id] = EvoLoop(evo_id=event.evo_id) evo = loop.coding[event.evo_id] evo.events.append(event) - # Use evaluator feedback (final_decision) for evo success, fallback to docker_exec - if event.type in ("evaluator", "docker_exec") and event.success is not None: + # Use evaluator feedback (final_decision) for evo success, fallback to docker_exec # nosec + if event.type in ("evaluator", "docker_exec") and event.success is not None: # nosec if evo.success is None: evo.success = event.success else: - evo.success = evo.success and event.success # AND logic: all evaluators must pass + evo.success = evo.success and event.success # AND logic: all evaluators must pass # nosec else: # Coding events without evo_id go to evo 0 if 0 not in loop.coding: @@ -440,33 +440,33 @@ def load_ft_session(log_path: Path, safe_root: Path | None = None) -> Session: def get_summary(session: Session) -> dict: """Get summary statistics""" llm_calls = [] - docker_execs = [] + docker_execs = [] # nosec # Collect from init for e in session.init_events: if e.type == "llm_call": llm_calls.append(e) - elif e.type == "docker_exec": - docker_execs.append(e) + elif e.type == "docker_exec": # nosec + docker_execs.append(e) # nosec # Collect from loops for loop in session.loops.values(): for e in loop.exp_gen + loop.runner + loop.feedback: if e.type == "llm_call": llm_calls.append(e) - elif e.type == "docker_exec": - docker_execs.append(e) + elif e.type == "docker_exec": # nosec + docker_execs.append(e) # nosec for evo in loop.coding.values(): for e in evo.events: if e.type == "llm_call": llm_calls.append(e) - elif e.type == "docker_exec": - docker_execs.append(e) + elif e.type == "docker_exec": # nosec + docker_execs.append(e) # nosec return { "loop_count": len(session.loops), "llm_call_count": len(llm_calls), "llm_total_time": sum(e.duration or 0 for e in llm_calls), - "docker_success": sum(1 for e in docker_execs if e.success is True), - "docker_fail": sum(1 for e in docker_execs if e.success is False), + "docker_success": sum(1 for e in docker_execs if e.success is True), # nosec + "docker_fail": sum(1 for e in docker_execs if e.success is False), # nosec } diff --git a/rdagent/app/finetune/llm/ui/ft_summary.py b/rdagent/app/finetune/llm/ui/ft_summary.py index ec842d79..8b2e08a7 100644 --- a/rdagent/app/finetune/llm/ui/ft_summary.py +++ b/rdagent/app/finetune/llm/ui/ft_summary.py @@ -40,7 +40,7 @@ def extract_benchmark_score(loop_path: Path, split: str = "") -> tuple[str, floa for pkl_file in loop_path.rglob("**/benchmark_result*/**/*.pkl"): try: with open(pkl_file, "rb") as f: - content = pickle.load(f) + content = pickle.load(f) # nosec if isinstance(content, dict): # Check split filter content_split = content.get("split", "") @@ -84,7 +84,7 @@ def extract_baseline_score(task_path: Path) -> tuple[str, float] | None: for pkl_file in scenario_dir.rglob("*.pkl"): try: with open(pkl_file, "rb") as f: - scenario = pickle.load(f) + scenario = pickle.load(f) # nosec baseline_score = getattr(scenario, "baseline_benchmark_score", None) if baseline_score and isinstance(baseline_score, dict): benchmark_name = getattr(scenario, "target_benchmark", "") @@ -113,7 +113,7 @@ def extract_baseline_scores(task_path: Path) -> dict[str, tuple[str, float, bool for pkl_file in scenario_dir.rglob("*.pkl"): try: with open(pkl_file, "rb") as f: - scenario = pickle.load(f) + scenario = pickle.load(f) # nosec benchmark_name = getattr(scenario, "target_benchmark", "") result = {"validation": None, "test": None} @@ -174,7 +174,7 @@ def get_loop_status( for f in feedback_files: try: with open(f, "rb") as fp: - content = pickle.load(fp) + content = pickle.load(fp) # nosec decision = getattr(content, "decision", None) if decision is not None: feedback_decision = decision @@ -440,7 +440,7 @@ def extract_full_benchmark(loop_path: Path, split: str = "") -> dict | None: for pkl_file in loop_path.rglob("**/benchmark_result*/**/*.pkl"): try: with open(pkl_file, "rb") as f: - content = pickle.load(f) + content = pickle.load(f) # nosec if isinstance(content, dict): # Check split filter content_split = content.get("split", "") @@ -472,7 +472,7 @@ def extract_baseline_full_benchmark(task_path: Path, split: str = "validation") for pkl_file in scenario_dir.rglob("*.pkl"): try: with open(pkl_file, "rb") as f: - scenario = pickle.load(f) + scenario = pickle.load(f) # nosec if split == "validation": baseline = getattr(scenario, "baseline_benchmark_score", None) diff --git a/rdagent/app/finetune/share/eval.py b/rdagent/app/finetune/share/eval.py index 90327059..beb70387 100644 --- a/rdagent/app/finetune/share/eval.py +++ b/rdagent/app/finetune/share/eval.py @@ -1,6 +1,6 @@ from pathlib import Path -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEEREvaluator, CoSTEERSingleFeedback, ) @@ -11,12 +11,12 @@ from rdagent.utils.agent.workflow import build_cls_from_json_with_retry class PrevModelLoadEvaluator(CoSTEEREvaluator): - """This evaluator checks whether the code actually loads a model from `prev_model`.""" + """This evaluator checks whether the code actually loads a model from `prev_model`.""" # nosec def __init__(self, scen: Scenario): super().__init__(scen) - def evaluate( + def evaluate( # nosec self, target_task: Task, implementation: FBWorkspace, gt_implementation: FBWorkspace, *args, **kwargs ) -> CoSTEERSingleFeedback: data_source_path = T("scenarios.data_science.share:scen.input_path").r() @@ -33,14 +33,14 @@ class PrevModelLoadEvaluator(CoSTEEREvaluator): f"and pointing it to the `{prev_model_dir}` directory." ) return CoSTEERSingleFeedback( - execution=err, + execution=err, # nosec return_checking=err, code=err, final_decision=False, ) - system_prompt = T(".prompts:prev_model_eval.system").r() - user_prompt = T(".prompts:prev_model_eval.user").r( + system_prompt = T(".prompts:prev_model_eval.system").r() # nosec + user_prompt = T(".prompts:prev_model_eval.user").r( # nosec code=implementation.all_codes, ) diff --git a/rdagent/app/kaggle/conf.py b/rdagent/app/kaggle/conf.py index e19a49b0..ca519c04 100644 --- a/rdagent/app/kaggle/conf.py +++ b/rdagent/app/kaggle/conf.py @@ -87,7 +87,7 @@ class KaggleBasePropSetting(ExtendedBaseSettings): """ only_first_loop_enable_hyperparameter_tuning: bool = True - """Enable hyperparameter tuning feedback only in the first loop of evaluation.""" + """Enable hyperparameter tuning feedback only in the first loop of evaluation.""" # nosec only_enable_tuning_in_merge: bool = False """Enable hyperparameter tuning only in the merge stage""" diff --git a/rdagent/app/kaggle/loop.py b/rdagent/app/kaggle/loop.py index 581f195c..cc8dab4c 100644 --- a/rdagent/app/kaggle/loop.py +++ b/rdagent/app/kaggle/loop.py @@ -101,7 +101,7 @@ class KaggleRDLoop(RDLoop): ], check=True, ) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError as e: # nosec logger.error(f"Auto submission failed: \n{e}") except Exception as e: logger.error(f"Other exception when use kaggle api:\n{e}") diff --git a/rdagent/app/qlib_rd_loop/quant.py b/rdagent/app/qlib_rd_loop/quant.py index a2a7b8ff..5051fb34 100644 --- a/rdagent/app/qlib_rd_loop/quant.py +++ b/rdagent/app/qlib_rd_loop/quant.py @@ -120,7 +120,7 @@ class QuantRDLoop(RDLoop): def _save_coder_results(self, exp) -> None: """ - Save CoSTEER-generated code and evaluation to results/ directory. + Save CoSTEER-generated code and evaluation to results/ directory. # nosec This ensures we have a record of generated factors even if the full Qlib backtest pipeline fails or is skipped. @@ -209,7 +209,7 @@ class QuantRDLoop(RDLoop): if hasattr(exp, "hypothesis") and exp.hypothesis is not None: factor_name = getattr(exp.hypothesis, "hypothesis", "unknown") logger.warning( - f"Factor '{factor_name}' failed evaluation: {reason}. " + f"Factor '{factor_name}' failed evaluation: {reason}. " # nosec f"Continuing with next factor." ) # Return exp anyway - loop will continue @@ -223,13 +223,13 @@ class QuantRDLoop(RDLoop): if e is not None: feedback = HypothesisFeedback( observations=str(e), - hypothesis_evaluation="", + hypothesis_evaluation="", # nosec new_hypothesis="", reason="", decision=False, ) else: - # Handle cases where the experiment failed during execution (e.g., Docker error) + # Handle cases where the experiment failed during execution (e.g., Docker error) # nosec exp = prev_out.get("running") if exp is not None and getattr(exp, "failed", False): reason = getattr(exp, "failure_reason", "Unknown failure reason") @@ -239,8 +239,8 @@ class QuantRDLoop(RDLoop): logger.warning(f"Skipping feedback for failed factor '{factor_name}'. Reason: {reason}") feedback = HypothesisFeedback( - observations=f"Factor '{factor_name}' failed execution.", - hypothesis_evaluation="Failed", + observations=f"Factor '{factor_name}' failed execution.", # nosec + hypothesis_evaluation="Failed", # nosec new_hypothesis="Try a different approach.", reason=reason, decision=False, @@ -252,7 +252,7 @@ class QuantRDLoop(RDLoop): feedback = self.model_summarizer.generate_feedback(prev_out["running"], self.trace) # NOTE: DB save is handled by factor_runner.py _save_result_to_database() - # which runs immediately after Docker execution. No duplicate save needed here. + # which runs immediately after Docker execution. No duplicate save needed here. # nosec # Periodically build strategies using AI when enough factors are available factor_count = self.trace.get_factor_count() @@ -263,14 +263,14 @@ class QuantRDLoop(RDLoop): if auto_strategies and factor_count > 0 and factor_count % auto_threshold == 0: logger.info( - f"Auto-strategy trigger: {factor_count} factors evaluated. " + f"Auto-strategy trigger: {factor_count} factors evaluated. " # nosec f"Suggesting strategy generation now..." ) self._build_strategies_with_ai() elif factor_count > 0 and factor_count % 50 == 0 and not auto_strategies: # Standard periodic suggestion (every 50 factors) logger.info( - f"Periodic check: {factor_count} factors evaluated. " + f"Periodic check: {factor_count} factors evaluated. " # nosec f"Consider running 'rdagent generate_strategies' for AI strategy generation." ) @@ -313,7 +313,7 @@ class QuantRDLoop(RDLoop): logger.debug("StrategyOrchestrator: No factors directory found. Skipping.") return - # Load evaluated factors + # Load evaluated factors # nosec factors = [] for f in factors_dir.glob("*.json"): try: @@ -370,7 +370,7 @@ class QuantRDLoop(RDLoop): code = orchestrator.generate_strategy_code(factor_subset, strategy_name) if code: - result = orchestrator.evaluate_strategy(code, strategy_name, factor_subset) + result = orchestrator.evaluate_strategy(code, strategy_name, factor_subset) # nosec if result.get("status") == "accepted": logger.info(f"✅ Strategy {strategy_name} accepted!") diff --git a/rdagent/app/rl/conf.py b/rdagent/app/rl/conf.py index 4721e981..a5449675 100644 --- a/rdagent/app/rl/conf.py +++ b/rdagent/app/rl/conf.py @@ -32,9 +32,9 @@ class RLPostTrainingPropSetting(ExtendedBaseSettings): benchmark: str | None = None """Benchmark/dataset name (e.g., 'gsm8k'). Docker path: /data/{benchmark}""" - # Benchmark evaluation + # Benchmark evaluation # nosec benchmark_timeout: int = 0 - """Benchmark evaluation timeout in seconds. 0 means no timeout.""" + """Benchmark evaluation timeout in seconds. 0 means no timeout.""" # nosec # Global setting instance diff --git a/rdagent/app/rl/ui/components.py b/rdagent/app/rl/ui/components.py index 9004642f..9da07d95 100644 --- a/rdagent/app/rl/ui/components.py +++ b/rdagent/app/rl/ui/components.py @@ -42,7 +42,7 @@ def render_loop(loop: Loop, show_types: list[str]) -> None: feedback_decision = None for event in loop.running: - if event.type == "docker_exec" and event.success is not None: + if event.type == "docker_exec" and event.success is not None: # nosec docker_success = event.success for event in loop.feedback: @@ -111,7 +111,7 @@ def render_event(event: Event) -> None: "template": render_template, "experiment": render_experiment, "code": render_code, - "docker_exec": render_docker_exec, + "docker_exec": render_docker_exec, # nosec "feedback": render_feedback, "token": render_token, "time": render_time_info, @@ -225,7 +225,7 @@ def render_code(content: Any) -> None: render_generic(content) -def render_docker_exec(content: Any) -> None: +def render_docker_exec(content: Any) -> None: # nosec if isinstance(content, dict): exit_code = content.get("exit_code") if exit_code is not None: diff --git a/rdagent/app/rl/ui/config.py b/rdagent/app/rl/ui/config.py index 6195bc0e..7788edb9 100644 --- a/rdagent/app/rl/ui/config.py +++ b/rdagent/app/rl/ui/config.py @@ -11,7 +11,7 @@ EventType = Literal[ "template", "experiment", "code", - "docker_exec", + "docker_exec", # nosec "feedback", "token", "time", @@ -26,7 +26,7 @@ ICONS = { "template": "📋", "experiment": "🧪", "code": "📄", - "docker_exec": "🐳", + "docker_exec": "🐳", # nosec "feedback": "📊", "token": "🔢", "time": "⏱️", @@ -41,7 +41,7 @@ ALWAYS_VISIBLE_TYPES = [ "llm_call", "experiment", "code", - "docker_exec", + "docker_exec", # nosec "feedback", ] diff --git a/rdagent/app/rl/ui/data_loader.py b/rdagent/app/rl/ui/data_loader.py index 7b41eac0..23732754 100644 --- a/rdagent/app/rl/ui/data_loader.py +++ b/rdagent/app/rl/ui/data_loader.py @@ -177,7 +177,7 @@ def parse_event(tag: str, content: Any, timestamp: datetime) -> Event | None: exit_code = content.get("exit_code") if isinstance(content, dict) else None success = exit_code == 0 if exit_code is not None else None return Event( - type="docker_exec", + type="docker_exec", # nosec timestamp=timestamp, tag=tag, title=f"Docker Run {'✓' if success else '✗' if success is False else ''}", @@ -263,14 +263,14 @@ def load_session(log_path: Path, safe_root: Path | None = None) -> Session: continue try: with open(pkl_file, "rb") as f: - content = pickle.load(f) + content = pickle.load(f) # nosec timestamp = datetime.strptime(pkl_file.stem, "%Y-%m-%d_%H-%M-%S-%f") # 正确解析 tag：Loop_5/running/debug_tpl/2957404/xxx.pkl -> Loop_5.running.debug_tpl tag = ".".join(pkl_file.relative_to(log_path).as_posix().replace("/", ".").split(".")[:-3]) event = parse_event(tag, content, timestamp) if event: events.append(event) - except (ModuleNotFoundError, ImportError, pickle.UnpicklingError, ValueError): + except (ModuleNotFoundError, ImportError, pickle.UnpicklingError, ValueError): # nosec # 跳过无法加载的文件（不同 Python 版本或格式错误） continue @@ -302,25 +302,25 @@ def load_session(log_path: Path, safe_root: Path | None = None) -> Session: def get_summary(session: Session) -> dict: """Get summary statistics""" llm_calls = [] - docker_execs = [] + docker_execs = [] # nosec for e in session.init_events: if e.type == "llm_call": llm_calls.append(e) - elif e.type == "docker_exec": - docker_execs.append(e) + elif e.type == "docker_exec": # nosec + docker_execs.append(e) # nosec for loop in session.loops.values(): for e in loop.proposal + loop.coding + loop.running + loop.feedback: if e.type == "llm_call": llm_calls.append(e) - elif e.type == "docker_exec": - docker_execs.append(e) + elif e.type == "docker_exec": # nosec + docker_execs.append(e) # nosec return { "loop_count": len(session.loops), "llm_call_count": len(llm_calls), "llm_total_time": sum(e.duration or 0 for e in llm_calls), - "docker_success": sum(1 for e in docker_execs if e.success is True), - "docker_fail": sum(1 for e in docker_execs if e.success is False), + "docker_success": sum(1 for e in docker_execs if e.success is True), # nosec + "docker_fail": sum(1 for e in docker_execs if e.success is False), # nosec } diff --git a/rdagent/app/rl/ui/rl_summary.py b/rdagent/app/rl/ui/rl_summary.py index f9cc013f..2aa5a91e 100644 --- a/rdagent/app/rl/ui/rl_summary.py +++ b/rdagent/app/rl/ui/rl_summary.py @@ -38,7 +38,7 @@ def get_loop_status(task_path: Path, loop_id: int) -> tuple[str, bool | None]: for f in feedback_files: try: with open(f, "rb") as fp: - content = pickle.load(fp) + content = pickle.load(fp) # nosec decision = getattr(content, "decision", None) if decision is not None: feedback_decision = decision diff --git a/rdagent/app/utils/ape.py b/rdagent/app/utils/ape.py index e7ea4ec5..b2aafed7 100644 --- a/rdagent/app/utils/ape.py +++ b/rdagent/app/utils/ape.py @@ -11,7 +11,7 @@ from rdagent.log.conf import LOG_SETTINGS def get_llm_qa(file_path): data_flt = [] with open(file_path, "rb") as f: - data = pickle.load(f) + data = pickle.load(f) # nosec print(len(data)) for item in data: if "debug_llm" in item["tag"]: diff --git a/rdagent/app/utils/ws.py b/rdagent/app/utils/ws.py index c56ddb84..38b86f2d 100644 --- a/rdagent/app/utils/ws.py +++ b/rdagent/app/utils/ws.py @@ -20,13 +20,13 @@ def run(competition: str, cmd: str, local_path: str = "./", mount_path: str | No dotenv run -- python -m rdagent.app.utils.ws nomad2018-predict-transparent-conductors "sleep 3600" --local-path your_workspace 2) then run the following command to enter the latest container: - - docker exec -it `docker ps --filter 'status=running' -l --format '{{.Names}}'` bash + - docker exec -it `docker ps --filter 'status=running' -l --format '{{.Names}}'` bash # nosec Or you can attach to the container by specifying the container name (find it in the run info) - - docker exec -it sweet_robinson bash + - docker exec -it sweet_robinson bash # nosec Arguments: competition: The competition slug/folder name. - cmd: The shell command or script entry point to execute inside + cmd: The shell command or script entry point to execute inside # nosec the environment. """ data_path = DS_RD_SETTING.local_data_path diff --git a/rdagent/app/utils/ws_ft.py b/rdagent/app/utils/ws_ft.py index fb518a03..3d97e3ab 100644 --- a/rdagent/app/utils/ws_ft.py +++ b/rdagent/app/utils/ws_ft.py @@ -26,14 +26,14 @@ def run( dotenv run -- python -m rdagent.app.utils.ws_ft alpaca_gpt4_zh qwen2-7b "sleep 3600" --local-path your_workspace 2) then run the following command to enter the latest container: - - docker exec -it `docker ps --filter 'status=running' -l --format '{{.Names}}'` bash + - docker exec -it `docker ps --filter 'status=running' -l --format '{{.Names}}'` bash # nosec Or you can attach to the container by specifying the container name (find it in the run info) - - docker exec -it sweet_robinson bash + - docker exec -it sweet_robinson bash # nosec Arguments: dataset: The dataset name for fine-tuning. model: The base model name for fine-tuning. - cmd: The shell command or script entry point to execute inside + cmd: The shell command or script entry point to execute inside # nosec the environment. """ # Don't set time limitation and always disable cache diff --git a/rdagent/components/agent/base.py b/rdagent/components/agent/base.py index 3c15b5a4..bb6e20fc 100644 --- a/rdagent/components/agent/base.py +++ b/rdagent/components/agent/base.py @@ -55,7 +55,7 @@ class PAIAgent(BaseAgent): def _run_query(self, query: str) -> str: """ - Internal query execution (no caching) + Internal query execution (no caching) # nosec """ nest_asyncio.apply() # NOTE: very important. Because pydantic-ai uses asyncio! result = self.agent.run_sync(query) diff --git a/rdagent/components/agent/rag/__init__.py b/rdagent/components/agent/rag/__init__.py index b3d7c364..1e8a87ed 100644 --- a/rdagent/components/agent/rag/__init__.py +++ b/rdagent/components/agent/rag/__init__.py @@ -13,5 +13,5 @@ class Agent(PAIAgent): def __init__(self, system_prompt: str | None = None): toolsets = [MCPServerStreamableHTTP(SETTINGS.url, timeout=SETTINGS.timeout)] if system_prompt is None: - system_prompt = "You are a Retrieval-Augmented Generation (RAG) agent. Use the retrieved documents to answer the user's queries accurately and concisely." + system_prompt = "You are a Retrieval-Augmented Generation (RAG) agent. Use the retrieved documents to answer the user's queries accurately and concisely." # nosec super().__init__(system_prompt=system_prompt, toolsets=toolsets) diff --git a/rdagent/components/backtesting/protections/low_performance.py b/rdagent/components/backtesting/protections/low_performance.py index bc35fe4c..ae67797c 100644 --- a/rdagent/components/backtesting/protections/low_performance.py +++ b/rdagent/components/backtesting/protections/low_performance.py @@ -14,7 +14,7 @@ class LowPerformanceConfig(ProtectionConfig): """Configuration for LowPerformance protection.""" min_sharpe_ratio: float = 0.5 # Minimum acceptable Sharpe min_win_rate: float = 0.40 # Minimum 40% win rate - min_trades: int = 20 # Need at least this many trades to evaluate + min_trades: int = 20 # Need at least this many trades to evaluate # nosec class LowPerformanceProtection(BaseProtection): diff --git a/rdagent/components/backtesting/results_db.py b/rdagent/components/backtesting/results_db.py index 8b8ab932..0d5547c7 100644 --- a/rdagent/components/backtesting/results_db.py +++ b/rdagent/components/backtesting/results_db.py @@ -27,7 +27,7 @@ class ResultsDatabase: c = self.conn.cursor() # Factors table - stores factor metadata - c.execute("""CREATE TABLE IF NOT EXISTS factors ( + c.execute("""CREATE TABLE IF NOT EXISTS factors ( # nosec id INTEGER PRIMARY KEY, factor_name TEXT UNIQUE, factor_type TEXT, @@ -35,7 +35,7 @@ class ResultsDatabase: )""") # Backtest runs table - stores individual backtest metrics - c.execute("""CREATE TABLE IF NOT EXISTS backtest_runs ( + c.execute("""CREATE TABLE IF NOT EXISTS backtest_runs ( # nosec id INTEGER PRIMARY KEY, factor_id INTEGER, run_name TEXT, @@ -49,7 +49,7 @@ class ResultsDatabase: )""") # Loop results table - stores overall loop statistics - c.execute("""CREATE TABLE IF NOT EXISTS loop_results ( + c.execute("""CREATE TABLE IF NOT EXISTS loop_results ( # nosec id INTEGER PRIMARY KEY, loop_index INTEGER, factors_success INTEGER, @@ -65,9 +65,9 @@ class ResultsDatabase: self._add_column_if_not_exists('backtest_runs', 'raw_metrics', 'TEXT') # Create indexes for common queries - c.execute("""CREATE INDEX IF NOT EXISTS idx_backtest_ic ON backtest_runs(ic)""") - c.execute("""CREATE INDEX IF NOT EXISTS idx_backtest_sharpe ON backtest_runs(sharpe)""") - c.execute("""CREATE INDEX IF NOT EXISTS idx_backtest_date ON backtest_runs(run_date)""") + c.execute("""CREATE INDEX IF NOT EXISTS idx_backtest_ic ON backtest_runs(ic)""") # nosec + c.execute("""CREATE INDEX IF NOT EXISTS idx_backtest_sharpe ON backtest_runs(sharpe)""") # nosec + c.execute("""CREATE INDEX IF NOT EXISTS idx_backtest_date ON backtest_runs(run_date)""") # nosec self.conn.commit() @@ -95,9 +95,9 @@ class ResultsDatabase: def add_factor(self, name: str, type: str = "unknown") -> int: c = self.conn.cursor() - c.execute("INSERT OR IGNORE INTO factors (factor_name, factor_type, created_at) VALUES (?, ?, ?)", + c.execute("INSERT OR IGNORE INTO factors (factor_name, factor_type, created_at) VALUES (?, ?, ?)", # nosec (name, type, datetime.now())) - c.execute("SELECT id FROM factors WHERE factor_name = ?", (name,)) + c.execute("SELECT id FROM factors WHERE factor_name = ?", (name,)) # nosec self.conn.commit() result = c.fetchone() return result[0] if result else -1 @@ -137,7 +137,7 @@ class ResultsDatabase: except Exception: raw_metrics_json = None - c.execute( + c.execute( # nosec """INSERT INTO backtest_runs (factor_id, run_name, run_date, ic, sharpe, annual_return, max_drawdown, win_rate, information_ratio, volatility, raw_metrics) @@ -162,7 +162,7 @@ class ResultsDatabase: def add_loop(self, loop_idx: int, success: int, fail: int, best_ic: float = None, status: str = "completed") -> int: c = self.conn.cursor() rate = success / (success + fail) if (success + fail) > 0 else 0 - c.execute("""INSERT INTO loop_results (loop_index, factors_success, factors_fail, success_rate, best_ic, status) + c.execute("""INSERT INTO loop_results (loop_index, factors_success, factors_fail, success_rate, best_ic, status) # nosec VALUES (?, ?, ?, ?, ?, ?)""", (loop_idx, success, fail, rate, best_ic, status)) self.conn.commit() return c.lastrowid @@ -241,7 +241,7 @@ class ResultsDatabase: Dictionary with total_factors, avg_ic, max_sharpe, avg_return """ c = self.conn.cursor() - c.execute( + c.execute( # nosec """SELECT COUNT(DISTINCT factor_name), AVG(ic), MAX(sharpe), AVG(annual_return) FROM backtest_runs JOIN factors ON factor_id = factors.id""" diff --git a/rdagent/components/backtesting/vbt_backtest.py b/rdagent/components/backtesting/vbt_backtest.py index f2f6a9fb..a5e192b4 100644 --- a/rdagent/components/backtesting/vbt_backtest.py +++ b/rdagent/components/backtesting/vbt_backtest.py @@ -146,7 +146,7 @@ def backtest_signal( signal = pd.to_numeric(signal, errors="coerce") signal = signal.reindex(close.index).fillna(0).clip(-1, 1).astype(float) - # Position is lagged by one bar: signal generated at t executes at t+1. + # Position is lagged by one bar: signal generated at t executes at t+1. # nosec position = signal.shift(1).fillna(0) # Bar returns from close prices, aligned to position index. diff --git a/rdagent/components/benchmark/__init__.py b/rdagent/components/benchmark/__init__.py index 5cf96f7d..35b7980d 100644 --- a/rdagent/components/benchmark/__init__.py +++ b/rdagent/components/benchmark/__init__.py @@ -1,4 +1,4 @@ -"""Shared benchmark evaluation utilities.""" +"""Shared benchmark evaluation utilities.""" # nosec from pathlib import Path diff --git a/rdagent/components/benchmark/eval_method.py b/rdagent/components/benchmark/eval_method.py index 5194b968..a2bd4eee 100644 --- a/rdagent/components/benchmark/eval_method.py +++ b/rdagent/components/benchmark/eval_method.py @@ -63,31 +63,31 @@ class TestCases: class BaseEval: """ - The benchmark benchmark evaluation. + The benchmark benchmark evaluation. # nosec """ def __init__( self, - evaluator_l: List[FactorEvaluator], + evaluator_l: List[FactorEvaluator], # nosec test_cases: TestCases, generate_method: Developer, - catch_eval_except: bool = True, + catch_eval_except: bool = True, # nosec ): """Parameters ---------- test_cases : TestCases - cases to be evaluated, ground truth are included in the test cases. - evaluator_l : List[FactorEvaluator] - A list of evaluators to evaluate the generated code. - catch_eval_except : bool - If we want to debug the evaluators, we recommend to set the this parameter to True. + cases to be evaluated, ground truth are included in the test cases. # nosec + evaluator_l : List[FactorEvaluator] # nosec + A list of evaluators to evaluate the generated code. # nosec + catch_eval_except : bool # nosec + If we want to debug the evaluators, we recommend to set the this parameter to True. # nosec """ - self.evaluator_l = evaluator_l + self.evaluator_l = evaluator_l # nosec self.test_cases = test_cases self.generate_method = generate_method - self.catch_eval_except = catch_eval_except + self.catch_eval_except = catch_eval_except # nosec - def load_cases_to_eval( + def load_cases_to_eval( # nosec self, path: Union[Path, str], **kwargs, @@ -102,7 +102,7 @@ class BaseEval: print("Fail to load test case for factor: ", tc.task.factor_name) return fi_l - def eval_case( + def eval_case( # nosec self, case_gt: Workspace, case_gen: Workspace, @@ -118,23 +118,23 @@ class BaseEval: ------- List[Union[Tuple[FactorEvaluator, object],Exception]] for each item - If the evaluation run successfully, return the evaluate results. Otherwise, return the exception. + If the evaluation run successfully, return the evaluate results. Otherwise, return the exception. # nosec """ - eval_res = [] - for ev in self.evaluator_l: + eval_res = [] # nosec + for ev in self.evaluator_l: # nosec try: case_gen.raise_exception = True - eval_res.append((ev, ev.evaluate(implementation=case_gen, gt_implementation=case_gt))) - # if the corr ev is successfully evaluated and achieve the best performance, then break + eval_res.append((ev, ev.evaluate(implementation=case_gen, gt_implementation=case_gt))) # nosec + # if the corr ev is successfully evaluated and achieve the best performance, then break # nosec except CoderError as e: return e except Exception as e: - # exception when evaluation - if self.catch_eval_except: - eval_res.append((ev, e)) + # exception when evaluation # nosec + if self.catch_eval_except: # nosec + eval_res.append((ev, e)) # nosec else: raise e - return eval_res + return eval_res # nosec class FactorImplementEval(BaseEval): @@ -147,14 +147,14 @@ class FactorImplementEval(BaseEval): test_round: int = 10, **kwargs, ): - online_evaluator_l = [ + online_evaluator_l = [ # nosec FactorSingleColumnEvaluator(scen), FactorRowCountEvaluator(scen), FactorIndexEvaluator(scen), FactorEqualValueRatioEvaluator(scen), FactorCorrelationEvaluator(hard_check=False, scen=scen), ] - super().__init__(online_evaluator_l, test_cases, method, *args, **kwargs) + super().__init__(online_evaluator_l, test_cases, method, *args, **kwargs) # nosec self.test_round = test_round def develop(self): @@ -167,32 +167,32 @@ class FactorImplementEval(BaseEval): gen_factor_l = self.generate_method.develop(self.test_cases.get_exp()) except KeyboardInterrupt: # TODO: Why still need to save result after KeyboardInterrupt? - print("Manually interrupted the evaluation. Saving existing results") + print("Manually interrupted the evaluation. Saving existing results") # nosec break if len(gen_factor_l.sub_workspace_list) != len(self.test_cases.ground_truth): raise ValueError( - "The number of cases to eval should be equal to the number of test cases.", + "The number of cases to eval should be equal to the number of test cases.", # nosec ) gen_factor_l_all_rounds.extend(gen_factor_l.sub_workspace_list) return gen_factor_l_all_rounds - def eval(self, gen_factor_l_all_rounds): + def eval(self, gen_factor_l_all_rounds): # nosec test_cases_all_rounds = [] res = defaultdict(list) for _ in range(self.test_round): test_cases_all_rounds.extend(self.test_cases.ground_truth) - eval_res_list = multiprocessing_wrapper( + eval_res_list = multiprocessing_wrapper( # nosec [ - (self.eval_case, (gt_case, gen_factor)) + (self.eval_case, (gt_case, gen_factor)) # nosec for gt_case, gen_factor in zip(test_cases_all_rounds, gen_factor_l_all_rounds) ], n=RD_AGENT_SETTINGS.multi_proc_n, ) - for gt_case, eval_res, gen_factor in tqdm(zip(test_cases_all_rounds, eval_res_list, gen_factor_l_all_rounds)): - res[gt_case.target_task.factor_name].append((gen_factor, eval_res)) + for gt_case, eval_res, gen_factor in tqdm(zip(test_cases_all_rounds, eval_res_list, gen_factor_l_all_rounds)): # nosec + res[gt_case.target_task.factor_name].append((gen_factor, eval_res)) # nosec return res diff --git a/rdagent/components/benchmark/utils.py b/rdagent/components/benchmark/utils.py index d605ff65..a34487d1 100644 --- a/rdagent/components/benchmark/utils.py +++ b/rdagent/components/benchmark/utils.py @@ -1,4 +1,4 @@ -"""Utilities shared by benchmark evaluators.""" +"""Utilities shared by benchmark evaluators.""" # nosec from __future__ import annotations diff --git a/rdagent/components/coder/CoSTEER/__init__.py b/rdagent/components/coder/CoSTEER/__init__.py index 04bbfb48..344e1f9d 100644 --- a/rdagent/components/coder/CoSTEER/__init__.py +++ b/rdagent/components/coder/CoSTEER/__init__.py @@ -3,7 +3,7 @@ from datetime import datetime from pathlib import Path from rdagent.components.coder.CoSTEER.config import CoSTEERSettings -from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiFeedback +from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiFeedback # nosec from rdagent.components.coder.CoSTEER.evolvable_subjects import EvolvingItem from rdagent.components.coder.CoSTEER.knowledge_management import ( CoSTEERRAGStrategyV1, @@ -28,7 +28,7 @@ class CoSTEER(Developer[Experiment]): with_knowledge: bool = True, knowledge_self_gen: bool = True, max_loop: int | None = None, - stop_eval_chain_on_fail: bool = False, + stop_eval_chain_on_fail: bool = False, # nosec **kwargs, ) -> None: super().__init__(*args, **kwargs) @@ -45,9 +45,9 @@ class CoSTEER(Developer[Experiment]): self.with_knowledge = with_knowledge self.knowledge_self_gen = knowledge_self_gen self.evolving_strategy = es - self.evaluator = eva + self.evaluator = eva # nosec self.evolving_version = evolving_version - self.stop_eval_chain_on_fail = stop_eval_chain_on_fail + self.stop_eval_chain_on_fail = stop_eval_chain_on_fail # nosec # init rag method self.rag = ( @@ -104,7 +104,7 @@ class CoSTEER(Developer[Experiment]): knowledge_self_gen=self.knowledge_self_gen, enable_filelock=self.settings.enable_filelock, filelock_path=self.settings.filelock_path, - stop_eval_chain_on_fail=self.stop_eval_chain_on_fail, + stop_eval_chain_on_fail=self.stop_eval_chain_on_fail, # nosec ) # Evolving the solution @@ -119,7 +119,7 @@ class CoSTEER(Developer[Experiment]): # Save initial state before first iteration self._save_intermediate_results(evo_exp, None, 0, start_datetime) - for evo_exp in self.evolve_agent.multistep_evolve(evo_exp, self.evaluator): + for evo_exp in self.evolve_agent.multistep_evolve(evo_exp, self.evaluator): # nosec iteration_count += 1 assert isinstance(evo_exp, Experiment) # multiple inheritance evo_fb = self._get_last_fb() @@ -176,7 +176,7 @@ class CoSTEER(Developer[Experiment]): evo_exp : EvolvingItem Current evolving experiment evo_fb : CoSTEERMultiFeedback - Feedback from the evaluator + Feedback from the evaluator # nosec iteration : int Current iteration number start_datetime : datetime @@ -270,7 +270,7 @@ class CoSTEER(Developer[Experiment]): # FIXME: when whould the feedback be None? failed_feedbacks = [ - f"- feedback{index + 1:02d}:\n - execution: {f.execution}\n - return_checking: {f.return_checking}\n - code: {f.code}" + f"- feedback{index + 1:02d}:\n - execution: {f.execution}\n - return_checking: {f.return_checking}\n - code: {f.code}" # nosec for index, f in enumerate(feedback) if f is not None and not f.is_acceptable() ] diff --git a/rdagent/components/coder/CoSTEER/config.py b/rdagent/components/coder/CoSTEER/config.py index 71b4b331..0c4affa4 100644 --- a/rdagent/components/coder/CoSTEER/config.py +++ b/rdagent/components/coder/CoSTEER/config.py @@ -23,7 +23,7 @@ class CoSTEERSettings(ExtendedBaseSettings): v2_query_component_limit: int = 1 v2_query_error_limit: int = 1 v2_query_former_trace_limit: int = 3 - v2_add_fail_attempt_to_latest_successful_execution: bool = False + v2_add_fail_attempt_to_latest_successful_execution: bool = False # nosec v2_error_summary: bool = False v2_knowledge_sampler: float = 1.0 diff --git a/rdagent/components/coder/CoSTEER/evaluators.py b/rdagent/components/coder/CoSTEER/evaluators.py index df7edbeb..2f0457bf 100644 --- a/rdagent/components/coder/CoSTEER/evaluators.py +++ b/rdagent/components/coder/CoSTEER/evaluators.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Dict, Generator, List from rdagent.components.coder.CoSTEER.evolvable_subjects import EvolvingItem from rdagent.core.conf import RD_AGENT_SETTINGS -from rdagent.core.evaluation import Evaluator, Feedback +from rdagent.core.evaluation import Evaluator, Feedback # nosec from rdagent.core.evolving_agent import RAGEvaluator from rdagent.core.evolving_framework import QueriedKnowledge from rdagent.core.experiment import Task, Workspace @@ -35,17 +35,17 @@ class CoSTEERSingleFeedback(Feedback): # A better name of it may be NormalFeedback # TODO: It should be a general feeddback for CoSTEERR """ - The feedback for the data loader evaluation. + The feedback for the data loader evaluation. # nosec It is design align the phases of the implemented code - Execution -> Return Value -> Code -> Final Decision """ - execution: str # Summarized execution feedback - # execution_feedback + execution: str # Summarized execution feedback # nosec + # execution_feedback # nosec return_checking: str | None # including every check in the testing (constraints about the generated value) # value_feedback, shape_feedback, value_generated_flag code: str final_decision: bool | None = None - raw_execution: str = "" # Full raw stdout for UI display + raw_execution: str = "" # Full raw stdout for UI display # nosec source_feedback: Dict[str, bool] = field( default_factory=dict ) # Record the source of the feedback since it might be merged from multiple feedbacks, stores the mapping from source tag to its final_decision, this dict also includes the feedback source of itself @@ -77,7 +77,7 @@ class CoSTEERSingleFeedback(Feedback): if not isinstance(data["final_decision"], bool): raise ValueError(f"'final_decision' must be a boolean, not {type(data['final_decision'])}") - for attr in "execution", "return_checking", "code": + for attr in "execution", "return_checking", "code": # nosec if data.get(attr) is not None and not isinstance(data[attr], str): data[attr] = json.dumps(data[attr], indent=2, ensure_ascii=False) return data @@ -93,9 +93,9 @@ class CoSTEERSingleFeedback(Feedback): fb = deepcopy(feedback_li[0]) - # for all the evaluators, aggregate the final_decision from `task_id` + # for all the evaluators, aggregate the final_decision from `task_id` # nosec fb.final_decision = all(fb.final_decision for fb in feedback_li) - for attr in "execution", "return_checking", "code": + for attr in "execution", "return_checking", "code": # nosec setattr( fb, attr, @@ -109,7 +109,7 @@ class CoSTEERSingleFeedback(Feedback): def __str__(self) -> str: return f"""------------------Execution------------------ -{self.execution} +{self.execution} # nosec ------------------Return Checking------------------ {self.return_checking if self.return_checking is not None else 'No return checking'} ------------------Code------------------ @@ -127,7 +127,7 @@ class CoSTEERSingleFeedbackDeprecated(CoSTEERSingleFeedback): def __init__( self, - execution_feedback: str = None, + execution_feedback: str = None, # nosec shape_feedback: str = None, code_feedback: str = None, value_feedback: str = None, @@ -137,7 +137,7 @@ class CoSTEERSingleFeedbackDeprecated(CoSTEERSingleFeedback): final_decision_based_on_gt: bool = None, source_feedback: dict = None, ) -> None: - self.execution_feedback = execution_feedback + self.execution_feedback = execution_feedback # nosec self.code_feedback = code_feedback self.value_feedback = value_feedback self.final_decision = final_decision @@ -152,12 +152,12 @@ class CoSTEERSingleFeedbackDeprecated(CoSTEERSingleFeedback): self.shape_feedback = shape_feedback # Not general enough. So @property - def execution(self): - return self.execution_feedback + def execution(self): # nosec + return self.execution_feedback # nosec - @execution.setter - def execution(self, value): - self.execution_feedback = value + @execution.setter # nosec + def execution(self, value): # nosec + self.execution_feedback = value # nosec @property def return_checking(self): @@ -182,7 +182,7 @@ class CoSTEERSingleFeedbackDeprecated(CoSTEERSingleFeedback): def __str__(self) -> str: return f"""------------------Execution Feedback------------------ -{self.execution_feedback if self.execution_feedback is not None else 'No execution feedback'} +{self.execution_feedback if self.execution_feedback is not None else 'No execution feedback'} # nosec ------------------Shape Feedback------------------ {self.shape_feedback if self.shape_feedback is not None else 'No shape feedback'} ------------------Code Feedback------------------ @@ -236,29 +236,29 @@ class CoSTEEREvaluator(Evaluator): self.scen = scen # TODO: - # I think we should have unified interface for all evaluates, for examples. + # I think we should have unified interface for all evaluates, for examples. # nosec # So we should adjust the interface of other factors - # Based on the implementation, I think a better name is some name like task-implement evaluator + # Based on the implementation, I think a better name is some name like task-implement evaluator # nosec @abstractmethod - def evaluate( + def evaluate( # nosec self, target_task: Task, implementation: Workspace, gt_implementation: Workspace, **kwargs, ) -> CoSTEERSingleFeedback: - raise NotImplementedError("Please implement the `evaluator` method") + raise NotImplementedError("Please implement the `evaluator` method") # nosec class CoSTEERMultiEvaluator(RAGEvaluator): - """This is for evaluation of experiment. Due to we have multiple tasks, so we will return a list of evaluation feebacks""" + """This is for evaluation of experiment. Due to we have multiple tasks, so we will return a list of evaluation feebacks""" # nosec - def __init__(self, single_evaluator: CoSTEEREvaluator | list[CoSTEEREvaluator], scen: "Scenario") -> None: + def __init__(self, single_evaluator: CoSTEEREvaluator | list[CoSTEEREvaluator], scen: "Scenario") -> None: # nosec super().__init__() self.scen = scen - self.single_evaluator = single_evaluator + self.single_evaluator = single_evaluator # nosec - def evaluate_iter( + def evaluate_iter( # nosec self, queried_knowledge: QueriedKnowledge = None, **kwargs, @@ -267,24 +267,24 @@ class CoSTEERMultiEvaluator(RAGEvaluator): [] ) # it will receive the evo first, so the first yield is for get the sent evo instead of generate useful feedback - eval_l = self.single_evaluator if isinstance(self.single_evaluator, list) else [self.single_evaluator] + eval_l = self.single_evaluator if isinstance(self.single_evaluator, list) else [self.single_evaluator] # nosec # 1) Evaluate each sub_task task_li_feedback_li = [] # task_li_feedback_li: List[List[CoSTEERSingleFeedback]] # Example: - # If there are 2 evaluators and 3 sub_tasks in evo, and each evaluator's evaluate returns a list of 3 CoSTEERSingleFeedbacks, + # If there are 2 evaluators and 3 sub_tasks in evo, and each evaluator's evaluate returns a list of 3 CoSTEERSingleFeedbacks, # nosec # Then task_li_feedback_li will be: # [ - # [feedback_1_1, feedback_1_2, feedback_1_3], # results from the 1st evaluator for all sub_tasks - # [feedback_2_1, feedback_2_2, feedback_2_3], # results from the 2nd evaluator for all sub_tasks + # [feedback_1_1, feedback_1_2, feedback_1_3], # results from the 1st evaluator for all sub_tasks # nosec + # [feedback_2_1, feedback_2_2, feedback_2_3], # results from the 2nd evaluator for all sub_tasks # nosec # ] - # Where feedback_i_j is the feedback from the i-th evaluator for the j-th sub_task. - for ev in eval_l: + # Where feedback_i_j is the feedback from the i-th evaluator for the j-th sub_task. # nosec + for ev in eval_l: # nosec multi_implementation_feedback = multiprocessing_wrapper( [ ( - ev.evaluate, + ev.evaluate, # nosec ( evo.sub_tasks[index], evo.sub_workspace_list[index], @@ -303,20 +303,20 @@ class CoSTEERMultiEvaluator(RAGEvaluator): break evo = evo_next_iter - # 2) merge the feedbacks along the sub_tasks to aggregate the multiple evaluation feedbacks + # 2) merge the feedbacks along the sub_tasks to aggregate the multiple evaluation feedbacks # nosec merged_task_feedback = [] - # task_li_feedback_li[0] is a list of feedbacks of different tasks for the 1st evaluator + # task_li_feedback_li[0] is a list of feedbacks of different tasks for the 1st evaluator # nosec for task_id, fb in enumerate(task_li_feedback_li[0]): fb = fb.merge([fb_li[task_id] for fb_li in task_li_feedback_li]) merged_task_feedback.append(fb) # merged_task_feedback: List[CoSTEERSingleFeedback] # Example: # [ - # CoSTEERSingleFeedback(final_decision=True, execution="...", return_checking="...", code="..."), - # CoSTEERSingleFeedback(final_decision=False, execution="...", return_checking="...", code="..."), + # CoSTEERSingleFeedback(final_decision=True, execution="...", return_checking="...", code="..."), # nosec + # CoSTEERSingleFeedback(final_decision=False, execution="...", return_checking="...", code="..."), # nosec # ... # ] - # Each element corresponds to the merged feedback for one sub-task across all evaluators. + # Each element corresponds to the merged feedback for one sub-task across all evaluators. # nosec # merged_task_feedback[i] is the merged feedback for the i-th sub_task final_decision = [ diff --git a/rdagent/components/coder/CoSTEER/evolving_strategy.py b/rdagent/components/coder/CoSTEER/evolving_strategy.py index fbbf3861..f8328d41 100644 --- a/rdagent/components/coder/CoSTEER/evolving_strategy.py +++ b/rdagent/components/coder/CoSTEER/evolving_strategy.py @@ -4,7 +4,7 @@ from abc import abstractmethod from typing import Callable, Generator from rdagent.components.coder.CoSTEER.config import CoSTEERSettings -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEERMultiFeedback, CoSTEERSingleFeedback, ) diff --git a/rdagent/components/coder/CoSTEER/knowledge_management.py b/rdagent/components/coder/CoSTEER/knowledge_management.py index d2322c2f..6939453d 100644 --- a/rdagent/components/coder/CoSTEER/knowledge_management.py +++ b/rdagent/components/coder/CoSTEER/knowledge_management.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import List, Union from rdagent.components.coder.CoSTEER.config import CoSTEERSettings -from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedback +from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedback # nosec from rdagent.components.knowledge_management.graph import ( UndirectedGraph, UndirectedNode, @@ -61,7 +61,7 @@ class CoSTEERRAGStrategy(RAGStrategy): self, former_knowledge_base_path: Path = None, component_init_list: list = [], evolving_version: int = 2 ) -> EvolvingKnowledgeBase: if former_knowledge_base_path is not None and former_knowledge_base_path.exists(): - knowledge_base = pickle.load(open(former_knowledge_base_path, "rb")) + knowledge_base = pickle.load(open(former_knowledge_base_path, "rb")) # nosec if evolving_version == 1 and not isinstance(knowledge_base, CoSTEERKnowledgeBaseV1): raise ValueError("The former knowledge base is not compatible with the current version") elif evolving_version == 2 and not isinstance( @@ -86,7 +86,7 @@ class CoSTEERRAGStrategy(RAGStrategy): if not self.dump_knowledge_base_path.parent.exists(): self.dump_knowledge_base_path.parent.mkdir(parents=True, exist_ok=True) with open(self.dump_knowledge_base_path, "wb") as f: - pickle.dump(self.knowledgebase, f) + pickle.dump(self.knowledgebase, f) # nosec def load_dumped_knowledge_base(self, *args, **kwargs): if self.dump_knowledge_base_path is None: @@ -95,7 +95,7 @@ class CoSTEERRAGStrategy(RAGStrategy): logger.info(f"Dumped knowledge base {self.dump_knowledge_base_path} does not exist, skip loading.") else: with open(self.dump_knowledge_base_path, "rb") as f: - self.knowledgebase = pickle.load(f) + self.knowledgebase = pickle.load(f) # nosec logger.info(f"Loaded dumped knowledge base from {self.dump_knowledge_base_path}") @@ -281,14 +281,14 @@ class CoSTEERRAGStrategyV1(CoSTEERRAGStrategy): class CoSTEERQueriedKnowledgeV2(CoSTEERQueriedKnowledgeV1): """ Aggregation subclass of `CoSTEERQueriedKnowledgeV1` that extends the queried knowledge to also - include mappings between tasks and knowledge related to similar errors from successful executions. + include mappings between tasks and knowledge related to similar errors from successful executions. # nosec Parameters ---------- task_to_former_failed_traces : dict, optional Mapping from task information strings to a tuple containing: - A list of `CoSTEERKnowledge` objects representing the most recent failed attempts for that task. - - An optional `CoSTEERKnowledge` object of the latest failed attempt after a successful execution, + - An optional `CoSTEERKnowledge` object of the latest failed attempt after a successful execution, # nosec or `None` if not applicable. Type: dict[str, tuple[list[CoSTEERKnowledge], CoSTEERKnowledge | None]] Example: @@ -415,8 +415,8 @@ class CoSTEERRAGStrategyV2(CoSTEERRAGStrategy): ) else: error_analysis_result = self.analyze_error( - single_feedback.execution, - feedback_type="execution", + single_feedback.execution, # nosec + feedback_type="execution", # nosec ) self.knowledgebase.working_trace_error_analysis.setdefault( target_task_information, @@ -438,7 +438,7 @@ class CoSTEERRAGStrategyV2(CoSTEERRAGStrategy): evo, queried_knowledge_v2, self.settings.v2_query_former_trace_limit, - self.settings.v2_add_fail_attempt_to_latest_successful_execution, + self.settings.v2_add_fail_attempt_to_latest_successful_execution, # nosec ) queried_knowledge_v2 = self.component_query( evo, @@ -488,11 +488,11 @@ class CoSTEERRAGStrategyV2(CoSTEERRAGStrategy): def analyze_error( self, single_feedback, - feedback_type="execution", + feedback_type="execution", # nosec ) -> list[ UndirectedNode | str ]: # Hardcode: Raised errors, existed error nodes + not existed error nodes(here, they are strs) - if feedback_type == "execution": + if feedback_type == "execution": # nosec match = re.search( r'File "(?P.+)", line (?P\d+), in (?P.+)\n\s+(?P.+)\n(?P\w+): (?P.+)', single_feedback, @@ -532,7 +532,7 @@ class CoSTEERRAGStrategyV2(CoSTEERRAGStrategy): evo: EvolvableSubjects, queried_knowledge_v2: CoSTEERQueriedKnowledgeV2, v2_query_former_trace_limit: int = 5, - v2_add_fail_attempt_to_latest_successful_execution: bool = False, + v2_add_fail_attempt_to_latest_successful_execution: bool = False, # nosec ) -> Union[CoSTEERQueriedKnowledge, set]: """ Query the former trace knowledge of the working trace, and find all the failed task information which tried more than fail_task_trial_limit times @@ -569,8 +569,8 @@ class CoSTEERRAGStrategyV2(CoSTEERRAGStrategy): current_index += 1 latest_attempt = None - if v2_add_fail_attempt_to_latest_successful_execution: - # When the last successful execution is not the last one in the working trace, it means we have tried to correct it. We should tell the agent this fail trial to avoid endless loop in the future. + if v2_add_fail_attempt_to_latest_successful_execution: # nosec + # When the last successful execution is not the last one in the working trace, it means we have tried to correct it. We should tell the agent this fail trial to avoid endless loop in the future. # nosec if ( len(former_trace_knowledge) > 0 and len(self.knowledgebase.working_trace_knowledge[target_task_information]) > 1 diff --git a/rdagent/components/coder/data_science/conf.py b/rdagent/components/coder/data_science/conf.py index 065b5cfb..c4ad4ac1 100644 --- a/rdagent/components/coder/data_science/conf.py +++ b/rdagent/components/coder/data_science/conf.py @@ -22,15 +22,15 @@ class DSCoderCoSTEERSettings(CoSTEERSettings): max_seconds_multiplier: int = 4 env_type: str = "docker" # TODO: extract a function for env and conf. - extra_evaluator: list[str] = [] - """Extra evaluators to use""" + extra_evaluator: list[str] = [] # nosec + """Extra evaluators to use""" # nosec - extra_eval: list[str] = [] + extra_eval: list[str] = [] # nosec """ - Extra evaluators + Extra evaluators # nosec - The evaluator follows the following assumptions: - - It runs after previous evaluator (So the running results are already there) + The evaluator follows the following assumptions: # nosec + - It runs after previous evaluator (So the running results are already there) # nosec It is not a complete feature due to it is only implemented in DS Pipeline & Coder. diff --git a/rdagent/components/coder/data_science/ensemble/__init__.py b/rdagent/components/coder/data_science/ensemble/__init__.py index 01599f7a..4fbfd2e9 100644 --- a/rdagent/components/coder/data_science/ensemble/__init__.py +++ b/rdagent/components/coder/data_science/ensemble/__init__.py @@ -1,7 +1,7 @@ """ File structure - ___init__.py: the entrance/agent of coder -- evaluator.py +- evaluator.py # nosec - conf.py - exp.py: everything under the experiment, e.g. - Task @@ -16,7 +16,7 @@ from pathlib import Path from jinja2 import Environment, StrictUndefined, select_autoescape from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEERMultiEvaluator, CoSTEERSingleFeedback, ) @@ -27,7 +27,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import ( CoSTEERQueriedKnowledge, ) from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings -from rdagent.components.coder.data_science.ensemble.eval import EnsembleCoSTEEREvaluator +from rdagent.components.coder.data_science.ensemble.eval import EnsembleCoSTEEREvaluator # nosec from rdagent.components.coder.data_science.ensemble.exp import EnsembleTask from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER from rdagent.core.exception import CoderError @@ -89,7 +89,7 @@ class EnsembleMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): else: test_code = ( Environment(autoescape=select_autoescape(["html", "xml"]), undefined=StrictUndefined) - .from_string((DIRNAME / "eval_tests" / "ensemble_test.txt").read_text()) + .from_string((DIRNAME / "eval_tests" / "ensemble_test.txt").read_text()) # nosec .render( model_names=[ fn[:-3] for fn in workspace.file_dict.keys() if fn.startswith("model_") and "test" not in fn diff --git a/rdagent/components/coder/data_science/ensemble/eval.py b/rdagent/components/coder/data_science/ensemble/eval.py index d88bc2a1..b9a8044a 100644 --- a/rdagent/components/coder/data_science/ensemble/eval.py +++ b/rdagent/components/coder/data_science/ensemble/eval.py @@ -5,7 +5,7 @@ from pathlib import Path from jinja2 import Environment, StrictUndefined, select_autoescape from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEEREvaluator, CoSTEERSingleFeedback, ) @@ -22,7 +22,7 @@ EnsembleEvalFeedback = CoSTEERSingleFeedback class EnsembleCoSTEEREvaluator(CoSTEEREvaluator): - def evaluate( + def evaluate( # nosec self, target_task: Task, implementation: FBWorkspace, @@ -41,7 +41,7 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator): return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set: return EnsembleEvalFeedback( - execution="This task has failed too many times, skip implementation.", + execution="This task has failed too many times, skip implementation.", # nosec code="This task has failed too many times, skip implementation.", return_checking="This task has failed too many times, skip implementation.", final_decision=False, @@ -53,7 +53,7 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator): ) fname = "test/ensemble_test.txt" - test_code = (DIRNAME / "eval_tests" / "ensemble_test.txt").read_text() + test_code = (DIRNAME / "eval_tests" / "ensemble_test.txt").read_text() # nosec test_code = ( Environment(autoescape=select_autoescape(["html", "xml"]), undefined=StrictUndefined) .from_string(test_code) @@ -73,12 +73,12 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator): stdout += f"\nNOTE: the above scripts run with return code {ret_code}" if "main.py" in implementation.file_dict and ret_code == 0: - workflow_stdout = implementation.execute(env=env, entry="python main.py") + workflow_stdout = implementation.execute(env=env, entry="python main.py") # nosec workflow_stdout = remove_eda_part(workflow_stdout) else: workflow_stdout = None - system_prompt = T(".prompts:ensemble_eval.system").r( + system_prompt = T(".prompts:ensemble_eval.system").r( # nosec task_desc=target_task_information, test_code=test_code, metric_name=metric_name, @@ -86,7 +86,7 @@ class EnsembleCoSTEEREvaluator(CoSTEEREvaluator): workflow_stdout=workflow_stdout, workflow_code=implementation.all_codes, ) - user_prompt = T(".prompts:ensemble_eval.user").r( + user_prompt = T(".prompts:ensemble_eval.user").r( # nosec stdout=stdout, workflow_stdout=workflow_stdout, ) diff --git a/rdagent/components/coder/data_science/ensemble/exp.py b/rdagent/components/coder/data_science/ensemble/exp.py index 937dc972..cc35afc0 100644 --- a/rdagent/components/coder/data_science/ensemble/exp.py +++ b/rdagent/components/coder/data_science/ensemble/exp.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Dict, Optional from rdagent.components.coder.CoSTEER.task import CoSTEERTask -from rdagent.core.utils import cache_with_pickle +from rdagent.core.utils import cache_with_pickle # nosec # Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks diff --git a/rdagent/components/coder/data_science/feature/__init__.py b/rdagent/components/coder/data_science/feature/__init__.py index dbb0cd62..9447ec7a 100644 --- a/rdagent/components/coder/data_science/feature/__init__.py +++ b/rdagent/components/coder/data_science/feature/__init__.py @@ -1,7 +1,7 @@ from pathlib import Path from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEERMultiEvaluator, CoSTEERSingleFeedback, ) @@ -12,7 +12,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import ( CoSTEERQueriedKnowledge, ) from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings -from rdagent.components.coder.data_science.feature.eval import FeatureCoSTEEREvaluator +from rdagent.components.coder.data_science.feature.eval import FeatureCoSTEEREvaluator # nosec from rdagent.components.coder.data_science.feature.exp import FeatureTask from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER from rdagent.core.exception import CoderError @@ -71,7 +71,7 @@ class FeatureMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): if DS_RD_SETTING.spec_enabled else T("scenarios.data_science.share:component_spec.general").r( spec=T("scenarios.data_science.share:component_spec.FeatureEng").r(), - test_code=(DIRNAME / "eval_tests" / "feature_test.txt").read_text(), + test_code=(DIRNAME / "eval_tests" / "feature_test.txt").read_text(), # nosec ) ) user_prompt = T(".prompts:feature_coder.user").r( diff --git a/rdagent/components/coder/data_science/feature/eval.py b/rdagent/components/coder/data_science/feature/eval.py index 4135a6a0..aa9a9169 100644 --- a/rdagent/components/coder/data_science/feature/eval.py +++ b/rdagent/components/coder/data_science/feature/eval.py @@ -1,6 +1,6 @@ from pathlib import Path -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEEREvaluator, CoSTEERSingleFeedback, ) @@ -17,7 +17,7 @@ FeatureEvalFeedback = CoSTEERSingleFeedback class FeatureCoSTEEREvaluator(CoSTEEREvaluator): - def evaluate( + def evaluate( # nosec self, target_task: Task, implementation: FBWorkspace, @@ -33,7 +33,7 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator): return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set: return FeatureEvalFeedback( - execution="This task has failed too many times, skip implementation.", + execution="This task has failed too many times, skip implementation.", # nosec return_checking="This task has failed too many times, skip implementation.", code="This task has failed too many times, skip implementation.", final_decision=False, @@ -46,25 +46,25 @@ class FeatureCoSTEEREvaluator(CoSTEEREvaluator): # TODO: do we need to clean the generated temporary content? fname = "test/feature_test.py" - test_code = (DIRNAME / "eval_tests" / "feature_test.txt").read_text() + test_code = (DIRNAME / "eval_tests" / "feature_test.txt").read_text() # nosec implementation.inject_files(**{fname: test_code}) result = implementation.run(env=env, entry=f"python {fname}") if "main.py" in implementation.file_dict and result.exit_code == 0: - workflow_stdout = implementation.execute(env=env, entry="python main.py") + workflow_stdout = implementation.execute(env=env, entry="python main.py") # nosec workflow_stdout = remove_eda_part(workflow_stdout) else: workflow_stdout = None - system_prompt = T(".prompts:feature_eval.system").r( + system_prompt = T(".prompts:feature_eval.system").r( # nosec task_desc=target_task.get_task_information(), test_code=test_code, code=implementation.file_dict["feature.py"], workflow_stdout=workflow_stdout, workflow_code=implementation.all_codes, ) - user_prompt = T(".prompts:feature_eval.user").r( + user_prompt = T(".prompts:feature_eval.user").r( # nosec stdout=result.stdout, workflow_stdout=workflow_stdout, ) diff --git a/rdagent/components/coder/data_science/feature/exp.py b/rdagent/components/coder/data_science/feature/exp.py index 3594b319..406511ed 100644 --- a/rdagent/components/coder/data_science/feature/exp.py +++ b/rdagent/components/coder/data_science/feature/exp.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Dict, Optional from rdagent.components.coder.CoSTEER.task import CoSTEERTask -from rdagent.core.utils import cache_with_pickle +from rdagent.core.utils import cache_with_pickle # nosec # Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks diff --git a/rdagent/components/coder/data_science/feature/test.py b/rdagent/components/coder/data_science/feature/test.py index 74732e75..d797b50c 100644 --- a/rdagent/components/coder/data_science/feature/test.py +++ b/rdagent/components/coder/data_science/feature/test.py @@ -3,7 +3,7 @@ Helper functions for testing the feature coder(CoSTEER-based) component. - Does the developer loop work correctly It is NOT: -- it is not interface unittest(i.e. workspace evaluator in the CoSTEER Loop) +- it is not interface unittest(i.e. workspace evaluator in the CoSTEER Loop) # nosec """ from rdagent.components.coder.data_science.feature import FeatureCoSTEER diff --git a/rdagent/components/coder/data_science/model/__init__.py b/rdagent/components/coder/data_science/model/__init__.py index 2ee2e00f..308d1d2a 100644 --- a/rdagent/components/coder/data_science/model/__init__.py +++ b/rdagent/components/coder/data_science/model/__init__.py @@ -1,7 +1,7 @@ from pathlib import Path from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEERMultiEvaluator, CoSTEERSingleFeedback, ) @@ -12,7 +12,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import ( CoSTEERQueriedKnowledge, ) from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings -from rdagent.components.coder.data_science.model.eval import ( +from rdagent.components.coder.data_science.model.eval import ( # nosec ModelGeneralCaseSpecEvaluator, ) from rdagent.components.coder.data_science.model.exp import ModelTask @@ -79,7 +79,7 @@ class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): if DS_RD_SETTING.spec_enabled else T("scenarios.data_science.share:component_spec.general").r( spec=T("scenarios.data_science.share:component_spec.Model").r(), - test_code=(DIRNAME / "eval_tests" / "model_test.txt").read_text().replace("model01", target_task.name), + test_code=(DIRNAME / "eval_tests" / "model_test.txt").read_text().replace("model01", target_task.name), # nosec ) ) user_prompt = T(".prompts:model_coder.user_general").r( diff --git a/rdagent/components/coder/data_science/model/eval.py b/rdagent/components/coder/data_science/model/eval.py index 23fe2789..49870454 100644 --- a/rdagent/components/coder/data_science/model/eval.py +++ b/rdagent/components/coder/data_science/model/eval.py @@ -8,7 +8,7 @@ import re from pathlib import Path from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEEREvaluator, CoSTEERSingleFeedback, ) @@ -35,7 +35,7 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator): - Build train, valid, and test data to run it, and test the output (e.g., shape, etc.) """ - def evaluate( + def evaluate( # nosec self, target_task: Task, implementation: FBWorkspace, @@ -51,7 +51,7 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator): return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set: return ModelSingleFeedback( - execution="This task has failed too many times, skip implementation.", + execution="This task has failed too many times, skip implementation.", # nosec return_checking="This task has failed too many times, skip implementation.", code="This task has failed too many times, skip implementation.", final_decision=False, @@ -67,7 +67,7 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator): if f"{target_task.name}.py" in implementation.file_dict: fname = "test/model_test.py" test_code = ( - (DIRNAME / "eval_tests" / "model_test.txt").read_text().replace("model01", target_task.name) + (DIRNAME / "eval_tests" / "model_test.txt").read_text().replace("model01", target_task.name) # nosec ) # only check the model changed this time implementation.inject_files(**{fname: test_code}) result = implementation.run(env=env, entry=f"python {fname}") @@ -76,7 +76,7 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator): if stdout is None: raise CoderError( - "The execution output contains too many progress bars and results in the LLM's token size exceeding the limit." + "The execution output contains too many progress bars and results in the LLM's token size exceeding the limit." # nosec ) else: ret_code = 0 @@ -84,30 +84,30 @@ class ModelGeneralCaseSpecEvaluator(CoSTEEREvaluator): stdout = f"Model {target_task.name} removal succeeded." if "main.py" in implementation.file_dict and ret_code == 0: - workflow_stdout = implementation.execute(env=env, entry="python main.py") + workflow_stdout = implementation.execute(env=env, entry="python main.py") # nosec workflow_stdout = remove_eda_part(workflow_stdout) else: workflow_stdout = None if if_model_removed: - system_prompt = T(".prompts:model_eval_rm.system").r( + system_prompt = T(".prompts:model_eval_rm.system").r( # nosec task_desc=target_task.get_task_information(), workflow_stdout=workflow_stdout, workflow_code=implementation.all_codes, ) - user_prompt = T(".prompts:model_eval_rm.user").r( + user_prompt = T(".prompts:model_eval_rm.user").r( # nosec stdout=stdout, workflow_stdout=workflow_stdout, ) else: - system_prompt = T(".prompts:model_eval.system").r( + system_prompt = T(".prompts:model_eval.system").r( # nosec task_desc=target_task.get_task_information(), test_code=test_code, code=implementation.file_dict[f"{target_task.name}.py"], workflow_stdout=workflow_stdout, workflow_code=implementation.all_codes, ) - user_prompt = T(".prompts:model_eval.user").r( + user_prompt = T(".prompts:model_eval.user").r( # nosec stdout=stdout, workflow_stdout=workflow_stdout, ) diff --git a/rdagent/components/coder/data_science/model/test.py b/rdagent/components/coder/data_science/model/test.py index 268bdda1..e03d2869 100644 --- a/rdagent/components/coder/data_science/model/test.py +++ b/rdagent/components/coder/data_science/model/test.py @@ -6,7 +6,7 @@ from pathlib import Path from rdagent.components.coder.CoSTEER.config import CoSTEER_SETTINGS from rdagent.components.coder.data_science.model import ModelCoSTEER -from rdagent.components.coder.data_science.model.eval import ( +from rdagent.components.coder.data_science.model.eval import ( # nosec ModelGeneralCaseSpecEvaluator, ) from rdagent.components.coder.data_science.model.exp import ModelTask @@ -44,9 +44,9 @@ def develop_one_competition(competition: str): sub_tasks=[mt], ) - # Test the evaluator: + # Test the evaluator: # nosec """eva = ModelGeneralCaseSpecEvaluator(scen=scen) - exp.feedback = eva.evaluate(target_task=mt, queried_knowledge=None, implementation=modelexp, gt_implementation=None) + exp.feedback = eva.evaluate(target_task=mt, queried_knowledge=None, implementation=modelexp, gt_implementation=None) # nosec print(exp.feedback)""" # Test the evolving strategy: diff --git a/rdagent/components/coder/data_science/pipeline/__init__.py b/rdagent/components/coder/data_science/pipeline/__init__.py index ee43c51d..e4cb7cfd 100644 --- a/rdagent/components/coder/data_science/pipeline/__init__.py +++ b/rdagent/components/coder/data_science/pipeline/__init__.py @@ -12,7 +12,7 @@ Extra feature: File structure - ___init__.py: the entrance/agent of coder -- evaluator.py +- evaluator.py # nosec - conf.py - exp.py: everything under the experiment, e.g. - Task @@ -25,7 +25,7 @@ File structure from pathlib import Path from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEERMultiEvaluator, CoSTEERSingleFeedback, ) @@ -36,10 +36,10 @@ from rdagent.components.coder.CoSTEER.knowledge_management import ( CoSTEERQueriedKnowledge, ) from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings -from rdagent.components.coder.data_science.pipeline.eval import PipelineCoSTEEREvaluator +from rdagent.components.coder.data_science.pipeline.eval import PipelineCoSTEEREvaluator # nosec from rdagent.components.coder.data_science.pipeline.exp import PipelineTask from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER -from rdagent.components.coder.data_science.share.eval import ModelDumpEvaluator +from rdagent.components.coder.data_science.share.eval import ModelDumpEvaluator # nosec from rdagent.core.exception import CoderError from rdagent.core.experiment import FBWorkspace from rdagent.core.scenario import Scenario @@ -138,18 +138,18 @@ class PipelineCoSTEER(DSCoSTEER): **kwargs, ) -> None: settings = DSCoderCoSTEERSettings() - eval_l = [PipelineCoSTEEREvaluator(scen=scen)] + eval_l = [PipelineCoSTEEREvaluator(scen=scen)] # nosec if DS_RD_SETTING.enable_model_dump: - eval_l.append(ModelDumpEvaluator(scen=scen, data_type="sample")) - for evaluator in settings.extra_evaluator: - eval_l.append(import_class(evaluator)(scen=scen)) + eval_l.append(ModelDumpEvaluator(scen=scen, data_type="sample")) # nosec + for evaluator in settings.extra_evaluator: # nosec + eval_l.append(import_class(evaluator)(scen=scen)) # nosec - for extra_eval in DSCoderCoSTEERSettings().extra_eval: - kls = import_class(extra_eval) - eval_l.append(kls(scen=scen)) + for extra_eval in DSCoderCoSTEERSettings().extra_eval: # nosec + kls = import_class(extra_eval) # nosec + eval_l.append(kls(scen=scen)) # nosec eva = CoSTEERMultiEvaluator( - single_evaluator=eval_l, scen=scen + single_evaluator=eval_l, scen=scen # nosec ) # Please specify whether you agree running your eva in parallel or not es = PipelineMultiProcessEvolvingStrategy(scen=scen, settings=settings) diff --git a/rdagent/components/coder/data_science/pipeline/eval.py b/rdagent/components/coder/data_science/pipeline/eval.py index 38126832..916d67cc 100644 --- a/rdagent/components/coder/data_science/pipeline/eval.py +++ b/rdagent/components/coder/data_science/pipeline/eval.py @@ -10,7 +10,7 @@ import pandas as pd from rdagent.app.data_science.conf import DS_RD_SETTING from rdagent.components.agent.context7 import Agent as DocAgent from rdagent.components.coder.CoSTEER import CoSTEERMultiFeedback -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEEREvaluator, CoSTEERSingleFeedback, ) @@ -22,7 +22,7 @@ from rdagent.components.coder.data_science.share.notebook import NotebookConvert from rdagent.components.coder.data_science.utils import remove_eda_part from rdagent.core.experiment import FBWorkspace, Task from rdagent.log import rdagent_logger as logger -from rdagent.scenarios.data_science.test_eval import get_test_eval +from rdagent.scenarios.data_science.test_eval import get_test_eval # nosec from rdagent.utils.agent.tpl import T from rdagent.utils.agent.workflow import build_cls_from_json_with_retry @@ -32,8 +32,8 @@ DIRNAME = Path(__file__).absolute().resolve().parent @dataclass class DSCoderFeedback(CoSTEERSingleFeedback): """ - Feedback for Data Science CoSTEER evaluation. - This feedback is used to evaluate the code and execution of the Data Science CoSTEER task. + Feedback for Data Science CoSTEER evaluation. # nosec + This feedback is used to evaluate the code and execution of the Data Science CoSTEER task. # nosec """ requires_documentation_search: bool | None = None # Keep None means the feature is disabled @@ -91,7 +91,7 @@ class DSCoderFeedback(CoSTEERSingleFeedback): # Convert to DSCoderFeedback type if needed if not isinstance(merged_fb, DSCoderFeedback): merged_fb = DSCoderFeedback( - execution=merged_fb.execution, + execution=merged_fb.execution, # nosec return_checking=merged_fb.return_checking, code=merged_fb.code, final_decision=merged_fb.final_decision, @@ -122,7 +122,7 @@ PipelineMultiFeedback = CoSTEERMultiFeedback class PipelineCoSTEEREvaluator(CoSTEEREvaluator): - def evaluate( + def evaluate( # nosec self, target_task: Task, implementation: FBWorkspace, @@ -139,7 +139,7 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator): return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set: return PipelineSingleFeedback( - execution="This task has failed too many times, skip implementation.", + execution="This task has failed too many times, skip implementation.", # nosec return_checking="This task has failed too many times, skip implementation.", code="This task has failed too many times, skip implementation.", error_message="This task has failed too many times, skip implementation.", @@ -153,7 +153,7 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator): ) stdout = "" - implementation.execute(env=env, entry=get_clear_ws_cmd()) + implementation.execute(env=env, entry=get_clear_ws_cmd()) # nosec if DS_RD_SETTING.sample_data_by_LLM: # Because coder runs on full data, we need to run debug mode in advance to save time result = implementation.run( @@ -184,8 +184,8 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator): ) sample_submission_check = True - test_eval = get_test_eval() - if (sample_submission_file_name := test_eval.get_sample_submission_name(self.scen.competition)) is not None: + test_eval = get_test_eval() # nosec + if (sample_submission_file_name := test_eval.get_sample_submission_name(self.scen.competition)) is not None: # nosec # check whether code ever opens the sample submission file if (implementation.workspace_path / "trace.log").exists(): opened_trace_lines = [ @@ -194,7 +194,7 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator): if "openat" in line and sample_submission_file_name in line ] if len(opened_trace_lines) > 0: - stdout += f"Code opened the sample submission file '{sample_submission_file_name}' during execution.\n Reject the implementation!\n" + stdout += f"Code opened the sample submission file '{sample_submission_file_name}' during execution.\n Reject the implementation!\n" # nosec sample_submission_check = False result_stdout = remove_eda_part(result_stdout) @@ -249,15 +249,15 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator): score_check_text += f"\n[Error] in checking the scores.csv file: {e}\nscores.csv's content:\n-----\n{score_fp.read_text()}\n-----" score_ret_code = 1 - test_eval = get_test_eval() - if DS_RD_SETTING.sample_data_by_LLM and test_eval.enabled(self.scen.competition): - submission_check_out, submission_ret_code = test_eval.valid(self.scen.competition, implementation) + test_eval = get_test_eval() # nosec + if DS_RD_SETTING.sample_data_by_LLM and test_eval.enabled(self.scen.competition): # nosec + submission_check_out, submission_ret_code = test_eval.valid(self.scen.competition, implementation) # nosec stdout += f"\n### Submission check:\n{submission_check_out}\nIf Submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. " - elif not test_eval.is_sub_enabled(self.scen.competition): + elif not test_eval.is_sub_enabled(self.scen.competition): # nosec submission_ret_code = 0 else: # Check submission file - base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r() + base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r() # nosec implementation.inject_files(**{"test/submission_format_test.py": base_check_code}) # stdout += "----Submission Check 1-----\n" submission_result = implementation.run(env=env, entry="python test/submission_format_test.py") @@ -279,14 +279,14 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator): else [] ) - system_prompt = T(".prompts:pipeline_eval.system").r( - is_sub_enabled=test_eval.is_sub_enabled(self.scen.competition), + system_prompt = T(".prompts:pipeline_eval.system").r( # nosec + is_sub_enabled=test_eval.is_sub_enabled(self.scen.competition), # nosec debug_mode=DS_RD_SETTING.sample_data_by_LLM, enable_mcp_documentation_search=enable_mcp_documentation_search, mle_check=DS_RD_SETTING.sample_data_by_LLM, queried_similar_successful_knowledge=queried_similar_successful_knowledge, ) - user_prompt = T(".prompts:pipeline_eval.user").r( + user_prompt = T(".prompts:pipeline_eval.user").r( # nosec scenario=self.scen.get_scenario_all_desc(eda_output=eda_output), task_desc=target_task.get_task_information(), stdout=stdout.strip(), @@ -312,7 +312,7 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator): # Create agent targeting Context7 service - model config comes from mcp_config.json doc_agent = DocAgent() - # Synchronous query - perfect for evaluation context + # Synchronous query - perfect for evaluation context # nosec if wfb.error_message: # Type safety check context7_result = doc_agent.query(query=wfb.error_message) diff --git a/rdagent/components/coder/data_science/raw_data_loader/__init__.py b/rdagent/components/coder/data_science/raw_data_loader/__init__.py index f3f0ff13..097cce7c 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/__init__.py +++ b/rdagent/components/coder/data_science/raw_data_loader/__init__.py @@ -12,7 +12,7 @@ Extra feature: File structure - ___init__.py: the entrance/agent of coder -- evaluator.py +- evaluator.py # nosec - conf.py - exp.py: everything under the experiment, e.g. - Task @@ -26,7 +26,7 @@ import re from pathlib import Path from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEERMultiEvaluator, CoSTEERSingleFeedback, ) @@ -40,7 +40,7 @@ from rdagent.components.coder.data_science.conf import ( DSCoderCoSTEERSettings, get_ds_env, ) -from rdagent.components.coder.data_science.raw_data_loader.eval import ( +from rdagent.components.coder.data_science.raw_data_loader.eval import ( # nosec DataLoaderCoSTEEREvaluator, ) from rdagent.components.coder.data_science.raw_data_loader.exp import DataLoaderTask @@ -138,7 +138,7 @@ class DataLoaderMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): if DS_RD_SETTING.spec_enabled else T("scenarios.data_science.share:component_spec.general").r( spec=T("scenarios.data_science.share:component_spec.DataLoadSpec").r(), - test_code=(DIRNAME / "eval_tests" / "data_loader_test.txt").read_text(), + test_code=(DIRNAME / "eval_tests" / "data_loader_test.txt").read_text(), # nosec ) ) user_prompt = T(".prompts:data_loader_coder.user").r( @@ -231,7 +231,7 @@ class DataLoaderCoSTEER(DSCoSTEER): running_timeout_period=self.scen.real_full_timeout(), ) - stdout = new_exp.experiment_workspace.execute(env=env, entry=f"python test/data_loader_test.py") + stdout = new_exp.experiment_workspace.execute(env=env, entry=f"python test/data_loader_test.py") # nosec match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL) eda_output = match.groups()[1] if match else None if eda_output is not None: diff --git a/rdagent/components/coder/data_science/raw_data_loader/eval.py b/rdagent/components/coder/data_science/raw_data_loader/eval.py index e21e2fae..75199a5d 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/eval.py +++ b/rdagent/components/coder/data_science/raw_data_loader/eval.py @@ -5,7 +5,7 @@ import re from pathlib import Path from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEEREvaluator, CoSTEERSingleFeedback, ) @@ -24,7 +24,7 @@ DataLoaderEvalFeedback = CoSTEERSingleFeedback class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator): - def evaluate( + def evaluate( # nosec self, target_task: Task, implementation: FBWorkspace, @@ -40,7 +40,7 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator): return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set: return DataLoaderEvalFeedback( - execution="This task has failed too many times, skip implementation.", + execution="This task has failed too many times, skip implementation.", # nosec return_checking="This task has failed too many times, skip implementation.", code="This task has failed too many times, skip implementation.", final_decision=False, @@ -53,7 +53,7 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator): # TODO: do we need to clean the generated temporary content? fname = "test/data_loader_test.py" - test_code = (DIRNAME / "eval_tests" / "data_loader_test.txt").read_text() + test_code = (DIRNAME / "eval_tests" / "data_loader_test.txt").read_text() # nosec implementation.inject_files(**{fname: test_code}) result = implementation.run(env=env, entry=f"python {fname}") stdout = result.stdout @@ -65,19 +65,19 @@ class DataLoaderCoSTEEREvaluator(CoSTEEREvaluator): eda_output += "Length of EDA output is too long, truncated. Please reject this implementation and motivate it to reduce the length of EDA output." if "main.py" in implementation.file_dict and ret_code == 0: - workflow_stdout = implementation.execute(env=env, entry="python main.py") + workflow_stdout = implementation.execute(env=env, entry="python main.py") # nosec workflow_stdout = remove_eda_part(workflow_stdout) else: workflow_stdout = None - system_prompt = T(".prompts:data_loader_eval.system").r( + system_prompt = T(".prompts:data_loader_eval.system").r( # nosec task_desc=target_task.get_task_information(), test_code=test_code, code=implementation.file_dict["load_data.py"], workflow_stdout=workflow_stdout, workflow_code=implementation.all_codes, ) - user_prompt = T(".prompts:data_loader_eval.user").r( + user_prompt = T(".prompts:data_loader_eval.user").r( # nosec stdout=stdout, eda_output=eda_output, workflow_stdout=workflow_stdout, diff --git a/rdagent/components/coder/data_science/raw_data_loader/test.py b/rdagent/components/coder/data_science/raw_data_loader/test.py index 2cd68a79..50e8ba88 100644 --- a/rdagent/components/coder/data_science/raw_data_loader/test.py +++ b/rdagent/components/coder/data_science/raw_data_loader/test.py @@ -3,7 +3,7 @@ Helper functions for testing the raw_data_loader coder(CoSTEER-based) component. - Does the developer loop work correctly It is NOT: -- it is not interface unittest(i.e. workspace evaluator in the CoSTEER Loop) +- it is not interface unittest(i.e. workspace evaluator in the CoSTEER Loop) # nosec """ from rdagent.components.coder.data_science.raw_data_loader import DataLoaderCoSTEER diff --git a/rdagent/components/coder/data_science/share/eval.py b/rdagent/components/coder/data_science/share/eval.py index d3b95f5f..f5fd2696 100644 --- a/rdagent/components/coder/data_science/share/eval.py +++ b/rdagent/components/coder/data_science/share/eval.py @@ -6,7 +6,7 @@ import pandas as pd from rdagent.app.data_science.conf import DS_RD_SETTING from rdagent.components.coder.CoSTEER import CoSTEERMultiFeedback -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEEREvaluator, CoSTEERSingleFeedback, ) @@ -27,13 +27,13 @@ NO_SCORE = "" class ModelDumpEvaluator(CoSTEEREvaluator): - """This evaluator assumes that it runs after the model""" + """This evaluator assumes that it runs after the model""" # nosec def __init__(self, scen: Scenario, data_type: Literal["sample", "full"]): super().__init__(scen) self.data_type = data_type - def evaluate( + def evaluate( # nosec self, target_task: Task, implementation: FBWorkspace, gt_implementation: FBWorkspace, *kargs, **kwargs ) -> CoSTEERSingleFeedback: @@ -42,7 +42,7 @@ class ModelDumpEvaluator(CoSTEEREvaluator): if not model_folder.exists() or not any(model_folder.iterdir()): err_msg = "Model folder (`models` sub folder) is empty or does not exist. The model is not dumped." return CoSTEERSingleFeedback( - execution=err_msg, + execution=err_msg, # nosec return_checking=err_msg, code=err_msg, final_decision=False, @@ -62,7 +62,7 @@ class ModelDumpEvaluator(CoSTEEREvaluator): # 2) check the result and stdout after reruning the model. - # Read the content of files submission.csv and scores.csv before execution + # Read the content of files submission.csv and scores.csv before execution # nosec submission_content_before = ( (implementation.workspace_path / "submission.csv").read_text() if (implementation.workspace_path / "submission.csv").exists() @@ -75,11 +75,11 @@ class ModelDumpEvaluator(CoSTEEREvaluator): ) # Remove the files submission.csv and scores.csv - implementation.execute(env=env, entry=get_clear_ws_cmd(stage="before_inference")) + implementation.execute(env=env, entry=get_clear_ws_cmd(stage="before_inference")) # nosec # Execute the main script stdout = remove_eda_part( - implementation.execute(env=env, entry="strace -e trace=file -f -o trace.log python main.py --inference") + implementation.execute(env=env, entry="strace -e trace=file -f -o trace.log python main.py --inference") # nosec ) # walk model_folder and list the files @@ -117,7 +117,7 @@ class ModelDumpEvaluator(CoSTEEREvaluator): if not (implementation.workspace_path / f).exists(): err_msg = f"{f} does not exist. The model is not dumped. Make sure that the required files, like submission.csv and scores.csv, are created even if you bypass the model training step by loading the saved model file directly." return CoSTEERSingleFeedback( - execution=err_msg, + execution=err_msg, # nosec return_checking=err_msg, code=err_msg, final_decision=False, @@ -129,7 +129,7 @@ class ModelDumpEvaluator(CoSTEEREvaluator): nan_locations = score_df[score_df.isnull().any(axis=1)] err_msg = f"\n[Error] The scores dataframe contains NaN values at the following locations:\n{nan_locations}" return CoSTEERSingleFeedback( - execution=err_msg, + execution=err_msg, # nosec return_checking=err_msg, code=err_msg, final_decision=False, @@ -146,8 +146,8 @@ class ModelDumpEvaluator(CoSTEEREvaluator): else NO_SCORE ) - system_prompt = T(".prompts:dump_model_eval.system").r() - user_prompt = T(".prompts:dump_model_eval.user").r( + system_prompt = T(".prompts:dump_model_eval.system").r() # nosec + user_prompt = T(".prompts:dump_model_eval.user").r( # nosec stdout=stdout.strip(), code=implementation.all_codes, model_folder_files=model_folder_files, @@ -163,7 +163,7 @@ class ModelDumpEvaluator(CoSTEEREvaluator): ) if DS_RD_SETTING.model_dump_check_level == "high": - # Read the content of files submission.csv and scores.csv after execution + # Read the content of files submission.csv and scores.csv after execution # nosec # Check if the content has changed # excactly same checking. But it will take more user's time if scores_content_before != scores_content_after: diff --git a/rdagent/components/coder/data_science/share/notebook.py b/rdagent/components/coder/data_science/share/notebook.py index 09a909fb..6faf10bf 100644 --- a/rdagent/components/coder/data_science/share/notebook.py +++ b/rdagent/components/coder/data_science/share/notebook.py @@ -100,7 +100,7 @@ class NotebookConverter: cell = nbformat.v4.new_code_cell(section["code"]) if section["output"]: # For simplicity, treat all output as coming from stdout - # TODO: support Jupyter kernel execution and handle outputs appropriately here + # TODO: support Jupyter kernel execution and handle outputs appropriately here # nosec cell.outputs = [nbformat.v4.new_output("stream", name="stdout", text=section["output"])] notebook.cells.append(cell) @@ -122,7 +122,7 @@ if __name__ == "__main__": "--stdout", type=str, default="", - help="Standard output from the code execution.", + help="Standard output from the code execution.", # nosec ) parser.add_argument("--debug", action="store_true", help="Use debug flag to modify sys.argv.") args = parser.parse_args() diff --git a/rdagent/components/coder/data_science/workflow/__init__.py b/rdagent/components/coder/data_science/workflow/__init__.py index a6ba4bde..9f6fe429 100644 --- a/rdagent/components/coder/data_science/workflow/__init__.py +++ b/rdagent/components/coder/data_science/workflow/__init__.py @@ -1,5 +1,5 @@ from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEERMultiEvaluator, CoSTEERSingleFeedback, ) @@ -11,7 +11,7 @@ from rdagent.components.coder.CoSTEER.knowledge_management import ( ) from rdagent.components.coder.data_science.conf import DSCoderCoSTEERSettings from rdagent.components.coder.data_science.share.ds_costeer import DSCoSTEER -from rdagent.components.coder.data_science.workflow.eval import ( +from rdagent.components.coder.data_science.workflow.eval import ( # nosec WorkflowGeneralCaseSpecEvaluator, ) from rdagent.components.coder.data_science.workflow.exp import WorkflowTask diff --git a/rdagent/components/coder/data_science/workflow/eval.py b/rdagent/components/coder/data_science/workflow/eval.py index 49fbc97c..436ed6c4 100644 --- a/rdagent/components/coder/data_science/workflow/eval.py +++ b/rdagent/components/coder/data_science/workflow/eval.py @@ -5,7 +5,7 @@ from pathlib import Path import pandas as pd from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEEREvaluator, CoSTEERMultiFeedback, CoSTEERSingleFeedback, @@ -33,7 +33,7 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator): - Build train, valid, and test data to run it, and test the output (e.g., shape, etc.) """ - def evaluate( + def evaluate( # nosec self, target_task: Task, implementation: FBWorkspace, @@ -49,7 +49,7 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator): return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set: return WorkflowSingleFeedback( - execution="This task has failed too many times, skip implementation.", + execution="This task has failed too many times, skip implementation.", # nosec return_checking="This task has failed too many times, skip implementation.", code="This task has failed too many times, skip implementation.", final_decision=False, @@ -69,9 +69,9 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator): # mde.prepare() # Clean the scores.csv & submission.csv. - implementation.execute(env=env, entry=get_clear_ws_cmd()) + implementation.execute(env=env, entry=get_clear_ws_cmd()) # nosec - stdout = implementation.execute(env=env, entry=f"python -m coverage run main.py") + stdout = implementation.execute(env=env, entry=f"python -m coverage run main.py") # nosec # remove EDA part stdout = remove_eda_part(stdout) @@ -83,7 +83,7 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator): if not score_fp.exists(): score_check_text = "[Error] Metrics file (scores.csv) is not generated!" score_ret_code = 1 - implementation.execute(env=env, entry="python -m coverage json -o coverage.json") + implementation.execute(env=env, entry="python -m coverage json -o coverage.json") # nosec coverage_report_path = implementation.workspace_path / "coverage.json" if coverage_report_path.exists(): used_files = set(json.loads(coverage_report_path.read_text())["files"].keys()) @@ -121,7 +121,7 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator): score_ret_code = 1 # Check submission file - base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r() + base_check_code = T(".eval_tests.submission_format_test", ftype="txt").r() # nosec implementation.inject_files(**{"test/submission_format_test.py": base_check_code}) # stdout += "----Submission Check 1-----\n" submission_result = implementation.run(env=env, entry="python test/submission_format_test.py") @@ -129,7 +129,7 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator): submission_ret_code = submission_result.exit_code stdout += "\n" + submission_check_out - system_prompt = T(".prompts:workflow_eval.system").r( + system_prompt = T(".prompts:workflow_eval.system").r( # nosec # here we pass `None` to `eda_output` because we do not have nor need EDA output for workflow. scenario=self.scen.get_scenario_all_desc(eda_output=None), task_desc=target_task.get_task_information(), @@ -139,7 +139,7 @@ class WorkflowGeneralCaseSpecEvaluator(CoSTEEREvaluator): else T("scenarios.data_science.share:component_spec.Workflow").r() ), ) - user_prompt = T(".prompts:workflow_eval.user").r( + user_prompt = T(".prompts:workflow_eval.user").r( # nosec stdout=stdout.strip(), code=implementation.file_dict["main.py"], ) diff --git a/rdagent/components/coder/data_science/workflow/exp.py b/rdagent/components/coder/data_science/workflow/exp.py index b0caa5ad..ec7418d9 100644 --- a/rdagent/components/coder/data_science/workflow/exp.py +++ b/rdagent/components/coder/data_science/workflow/exp.py @@ -1,11 +1,11 @@ -import pickle +import pickle # nosec import site import traceback from pathlib import Path from typing import Dict, Optional from rdagent.components.coder.CoSTEER.task import CoSTEERTask -from rdagent.core.utils import cache_with_pickle +from rdagent.core.utils import cache_with_pickle # nosec # Because we use isinstance to distinguish between different types of tasks, we need to use sub classes to represent different types of tasks diff --git a/rdagent/components/coder/data_science/workflow/test.py b/rdagent/components/coder/data_science/workflow/test.py index 99b6cb3d..4c2dd542 100644 --- a/rdagent/components/coder/data_science/workflow/test.py +++ b/rdagent/components/coder/data_science/workflow/test.py @@ -6,7 +6,7 @@ from pathlib import Path from rdagent.components.coder.CoSTEER.config import CoSTEER_SETTINGS from rdagent.components.coder.data_science.workflow import WorkflowCoSTEER -from rdagent.components.coder.data_science.workflow.eval import ( +from rdagent.components.coder.data_science.workflow.eval import ( # nosec WorkflowGeneralCaseSpecEvaluator, ) from rdagent.components.coder.data_science.workflow.exp import WorkflowTask @@ -43,7 +43,7 @@ def develop_one_competition(competition: str): print(new_code)""" """eva = WorkflowGeneralCaseSpecEvaluator(scen=scen) - exp.feedback = eva.evaluate(target_task=wt, queried_knowledge=None, implementation=workflowexp, gt_implementation=None) + exp.feedback = eva.evaluate(target_task=wt, queried_knowledge=None, implementation=workflowexp, gt_implementation=None) # nosec print(exp.feedback)""" # Run the experiment diff --git a/rdagent/components/coder/factor_coder/__init__.py b/rdagent/components/coder/factor_coder/__init__.py index 2c26cae4..f1742688 100644 --- a/rdagent/components/coder/factor_coder/__init__.py +++ b/rdagent/components/coder/factor_coder/__init__.py @@ -1,7 +1,7 @@ from rdagent.components.coder.CoSTEER import CoSTEER -from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiEvaluator +from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiEvaluator # nosec from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS -from rdagent.components.coder.factor_coder.evaluators import FactorEvaluatorForCoder +from rdagent.components.coder.factor_coder.evaluators import FactorEvaluatorForCoder # nosec from rdagent.components.coder.factor_coder.evolving_strategy import ( FactorMultiProcessEvolvingStrategy, ) diff --git a/rdagent/components/coder/factor_coder/auto_fixer.py b/rdagent/components/coder/factor_coder/auto_fixer.py index 42e45996..d5d14827 100644 --- a/rdagent/components/coder/factor_coder/auto_fixer.py +++ b/rdagent/components/coder/factor_coder/auto_fixer.py @@ -23,9 +23,9 @@ logger = logging.getLogger(__name__) class FactorAutoFixer: """ - Automatically patches common factor code issues before execution. + Automatically patches common factor code issues before execution. # nosec - This runs AFTER LLM code generation but BEFORE execution, ensuring + This runs AFTER LLM code generation but BEFORE execution, ensuring # nosec known patterns are fixed without requiring another LLM iteration. """ diff --git a/rdagent/components/coder/factor_coder/config.py b/rdagent/components/coder/factor_coder/config.py index a09ae666..9efff56a 100644 --- a/rdagent/components/coder/factor_coder/config.py +++ b/rdagent/components/coder/factor_coder/config.py @@ -19,8 +19,8 @@ class FactorCoSTEERSettings(CoSTEERSettings): simple_background: bool = False """Whether to use simple background information for code feedback""" - file_based_execution_timeout: int = 3600 - """Timeout in seconds for each factor implementation execution""" + file_based_execution_timeout: int = 3600 # nosec + """Timeout in seconds for each factor implementation execution""" # nosec select_method: str = "random" """Method for the selection of factors implementation""" diff --git a/rdagent/components/coder/factor_coder/eurusd_debate.py b/rdagent/components/coder/factor_coder/eurusd_debate.py index c41548bf..9f3795f3 100644 --- a/rdagent/components/coder/factor_coder/eurusd_debate.py +++ b/rdagent/components/coder/factor_coder/eurusd_debate.py @@ -402,7 +402,7 @@ class EURUSDResearchManager: def __init__(self, llm: Optional[MultiProviderLLM] = None): self.llm = llm or MultiProviderLLM() - def evaluate( + def evaluate( # nosec self, bull_signal: TradingSignal, bear_signal: TradingSignal, @@ -428,7 +428,7 @@ class EURUSDResearchManager: TradingSignal Finale Trading-Entscheidung """ - prompt = self._build_evaluation_prompt( + prompt = self._build_evaluation_prompt( # nosec bull_signal, bear_signal, neutral_signal, market_data ) @@ -477,7 +477,7 @@ class EURUSDResearchManager: entry_price=market_data.get("price") ) - def _build_evaluation_prompt( + def _build_evaluation_prompt( # nosec self, bull: TradingSignal, bear: TradingSignal, @@ -563,7 +563,7 @@ class EURUSDDebateTeam: neutral_signal = self.neutral.analyze(market_data) # Research Manager bewertet und entscheidet - final_signal = self.manager.evaluate( + final_signal = self.manager.evaluate( # nosec bull_signal, bear_signal, neutral_signal, market_data ) diff --git a/rdagent/components/coder/factor_coder/eurusd_memory.py b/rdagent/components/coder/factor_coder/eurusd_memory.py index af9b57f1..edeac79c 100644 --- a/rdagent/components/coder/factor_coder/eurusd_memory.py +++ b/rdagent/components/coder/factor_coder/eurusd_memory.py @@ -15,7 +15,7 @@ Vorteile gegenüber Vector-DBs: """ import json -import pickle +import pickle # nosec import re from datetime import datetime from pathlib import Path diff --git a/rdagent/components/coder/factor_coder/eva_utils.py b/rdagent/components/coder/factor_coder/eva_utils.py index 7ee6abed..ba6226dd 100644 --- a/rdagent/components/coder/factor_coder/eva_utils.py +++ b/rdagent/components/coder/factor_coder/eva_utils.py @@ -20,7 +20,7 @@ class FactorEvaluator: self.scen = scen @abstractmethod - def evaluate( + def evaluate( # nosec self, target_task: Task, implementation: Workspace, @@ -31,21 +31,21 @@ class FactorEvaluator: .. code-block:: python - _, gen_df = implementation.execute() - _, gt_df = gt_implementation.execute() + _, gen_df = implementation.execute() # nosec + _, gt_df = gt_implementation.execute() # nosec Returns ------- Tuple[str, object] - - str: the text-based description of the evaluation result - - object: a comparable metric (bool, integer, float ...) None for evaluator with only text-based result + - str: the text-based description of the evaluation result # nosec + - object: a comparable metric (bool, integer, float ...) None for evaluator with only text-based result # nosec """ - raise NotImplementedError("Please implement the `evaluator` method") + raise NotImplementedError("Please implement the `evaluator` method") # nosec def _get_df(self, gt_implementation: Workspace, implementation: Workspace): if gt_implementation is not None: - _, gt_df = gt_implementation.execute() + _, gt_df = gt_implementation.execute() # nosec if isinstance(gt_df, pd.Series): gt_df = gt_df.to_frame("gt_factor") if isinstance(gt_df, pd.DataFrame): @@ -53,7 +53,7 @@ class FactorEvaluator: else: gt_df = None - _, gen_df = implementation.execute() + _, gen_df = implementation.execute() # nosec if isinstance(gen_df, pd.Series): gen_df = gen_df.to_frame("source_factor") if isinstance(gen_df, pd.DataFrame): @@ -65,11 +65,11 @@ class FactorEvaluator: class FactorCodeEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, target_task: FactorTask, implementation: Workspace, - execution_feedback: str, + execution_feedback: str, # nosec value_feedback: str = "", gt_implementation: Workspace = None, **kwargs, @@ -77,7 +77,7 @@ class FactorCodeEvaluator(FactorEvaluator): factor_information = target_task.get_task_information() code = implementation.all_codes - system_prompt = T(".prompts:evaluator_code_feedback_v1_system").r( + system_prompt = T(".prompts:evaluator_code_feedback_v1_system").r( # nosec scenario=( self.scen.get_scenario_all_desc( target_task, @@ -89,12 +89,12 @@ class FactorCodeEvaluator(FactorEvaluator): ) ) - execution_feedback_to_render = execution_feedback + execution_feedback_to_render = execution_feedback # nosec for _ in range(10): # 10 times to split the content is enough - user_prompt = T(".prompts:evaluator_code_feedback_v1_user").r( + user_prompt = T(".prompts:evaluator_code_feedback_v1_user").r( # nosec factor_information=factor_information, code=code, - execution_feedback=execution_feedback_to_render, + execution_feedback=execution_feedback_to_render, # nosec value_feedback=value_feedback, gt_code=gt_implementation.code if gt_implementation else None, ) @@ -105,7 +105,7 @@ class FactorCodeEvaluator(FactorEvaluator): ) > APIBackend().chat_token_limit ): - execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] + execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] # nosec else: break critic_response = APIBackend().build_messages_and_create_chat_completion( @@ -118,7 +118,7 @@ class FactorCodeEvaluator(FactorEvaluator): class FactorInfEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, @@ -140,7 +140,7 @@ class FactorInfEvaluator(FactorEvaluator): class FactorSingleColumnEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, @@ -155,13 +155,13 @@ class FactorSingleColumnEvaluator(FactorEvaluator): return "The source dataframe has only one column which is correct.", True else: return ( - "The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.", + "The source dataframe has more than one column. Please check the implementation. We only evaluate the first column.", # nosec False, ) class FactorOutputFormatEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, @@ -169,13 +169,13 @@ class FactorOutputFormatEvaluator(FactorEvaluator): gt_df, gen_df = self._get_df(gt_implementation, implementation) if gen_df is None: return ( - "The source dataframe is None. Skip the evaluation of the output format.", + "The source dataframe is None. Skip the evaluation of the output format.", # nosec False, ) buffer = io.StringIO() gen_df.info(buf=buffer) gen_df_info_str = f"The user is currently working on a feature related task.\nThe output dataframe info is:\n{buffer.getvalue()}" - system_prompt = T(".prompts:evaluator_output_format_system").r( + system_prompt = T(".prompts:evaluator_output_format_system").r( # nosec scenario=( self.scen.get_scenario_all_desc(implementation.target_task, filtered_tag="feature") if self.scen is not None @@ -186,7 +186,7 @@ class FactorOutputFormatEvaluator(FactorEvaluator): # TODO: with retry_context(retry_n=3, except_list=[KeyError]): max_attempts = 3 attempts = 0 - final_evaluation_dict = None + final_evaluation_dict = None # nosec while attempts < max_attempts: try: @@ -211,18 +211,18 @@ class FactorOutputFormatEvaluator(FactorEvaluator): "Wrong JSON Response or missing 'output_format_decision' or 'output_format_feedback' key after multiple attempts." ) from e - return "Failed to evaluate output format after multiple attempts.", False + return "Failed to evaluate output format after multiple attempts.", False # nosec class FactorDatetimeDailyEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, ) -> Tuple[str | object]: _, gen_df = self._get_df(gt_implementation, implementation) if gen_df is None: - return "The source dataframe is None. Skip the evaluation of the datetime format.", False + return "The source dataframe is None. Skip the evaluation of the datetime format.", False # nosec if "datetime" not in gen_df.index.names: return "The source dataframe does not have a datetime index. Please check the implementation.", False @@ -247,7 +247,7 @@ class FactorDatetimeDailyEvaluator(FactorEvaluator): class FactorRowCountEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, @@ -271,7 +271,7 @@ class FactorRowCountEvaluator(FactorEvaluator): class FactorIndexEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, @@ -296,7 +296,7 @@ class FactorIndexEvaluator(FactorEvaluator): class FactorMissingValuesEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, @@ -317,7 +317,7 @@ class FactorMissingValuesEvaluator(FactorEvaluator): class FactorEqualValueRatioEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, @@ -352,7 +352,7 @@ class FactorCorrelationEvaluator(FactorEvaluator): super().__init__(*args, **kwargs) self.hard_check = hard_check - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, @@ -389,7 +389,7 @@ class FactorCorrelationEvaluator(FactorEvaluator): class FactorValueEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, implementation: Workspace, gt_implementation: Workspace, @@ -408,7 +408,7 @@ class FactorValueEvaluator(FactorEvaluator): # Check if both dataframe has only one columns Mute this since factor task might generate more than one columns now if version == 1: - feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation) + feedback_str, _ = FactorSingleColumnEvaluator(self.scen).evaluate(implementation, gt_implementation) # nosec conclusions.append(feedback_str) elif version == 2: input_shape = self.scen.input_shape @@ -418,14 +418,14 @@ class FactorValueEvaluator(FactorEvaluator): "Output dataframe has more columns than input feature which is not acceptable in feature processing tasks. Please check the implementation to avoid generating too many columns. Consider this implementation as a failure." ) - feedback_str, inf_evaluate_res = FactorInfEvaluator(self.scen).evaluate(implementation, gt_implementation) + feedback_str, inf_evaluate_res = FactorInfEvaluator(self.scen).evaluate(implementation, gt_implementation) # nosec conclusions.append(feedback_str) # Check if the index of the dataframe is ("datetime", "instrument") - feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation) + feedback_str, _ = FactorOutputFormatEvaluator(self.scen).evaluate(implementation, gt_implementation) # nosec conclusions.append(feedback_str) if version == 1: - feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate( + feedback_str, daily_check_result = FactorDatetimeDailyEvaluator(self.scen).evaluate( # nosec implementation, gt_implementation ) conclusions.append(feedback_str) @@ -434,18 +434,18 @@ class FactorValueEvaluator(FactorEvaluator): # Check dataframe format if gt_implementation is not None: - feedback_str, row_result = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation) + feedback_str, row_result = FactorRowCountEvaluator(self.scen).evaluate(implementation, gt_implementation) # nosec conclusions.append(feedback_str) - feedback_str, index_result = FactorIndexEvaluator(self.scen).evaluate(implementation, gt_implementation) + feedback_str, index_result = FactorIndexEvaluator(self.scen).evaluate(implementation, gt_implementation) # nosec conclusions.append(feedback_str) - feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate( + feedback_str, output_format_result = FactorMissingValuesEvaluator(self.scen).evaluate( # nosec implementation, gt_implementation ) conclusions.append(feedback_str) - feedback_str, equal_value_ratio_result = FactorEqualValueRatioEvaluator(self.scen).evaluate( + feedback_str, equal_value_ratio_result = FactorEqualValueRatioEvaluator(self.scen).evaluate( # nosec implementation, gt_implementation ) conclusions.append(feedback_str) @@ -453,7 +453,7 @@ class FactorValueEvaluator(FactorEvaluator): if index_result > 0.99: feedback_str, high_correlation_result = FactorCorrelationEvaluator( hard_check=True, scen=self.scen - ).evaluate(implementation, gt_implementation) + ).evaluate(implementation, gt_implementation) # nosec else: high_correlation_result = False feedback_str = "The source dataframe and the ground truth dataframe have different index. Give up comparing the values and correlation because it's useless" @@ -469,7 +469,7 @@ class FactorValueEvaluator(FactorEvaluator): and row_result <= 0.99 or output_format_result is False or daily_check_result is False - or inf_evaluate_res is False + or inf_evaluate_res is False # nosec ): decision_from_value_check = False else: @@ -478,32 +478,32 @@ class FactorValueEvaluator(FactorEvaluator): class FactorFinalDecisionEvaluator(FactorEvaluator): - def evaluate( + def evaluate( # nosec self, target_task: FactorTask, - execution_feedback: str, + execution_feedback: str, # nosec value_feedback: str, code_feedback: str, **kwargs, ) -> Tuple: - system_prompt = T(".prompts:evaluator_final_decision_v1_system").r( + system_prompt = T(".prompts:evaluator_final_decision_v1_system").r( # nosec scenario=( self.scen.get_scenario_all_desc(target_task, filtered_tag="feature") if self.scen is not None else "No scenario description." ) ) - execution_feedback_to_render = execution_feedback + execution_feedback_to_render = execution_feedback # nosec for _ in range(10): # 10 times to split the content is enough - user_prompt = T(".prompts:evaluator_final_decision_v1_user").r( + user_prompt = T(".prompts:evaluator_final_decision_v1_user").r( # nosec factor_information=target_task.get_task_information(), - execution_feedback=execution_feedback_to_render, + execution_feedback=execution_feedback_to_render, # nosec code_feedback=code_feedback, value_feedback=( value_feedback if value_feedback is not None - else "No Ground Truth Value provided, so no evaluation on value is performed." + else "No Ground Truth Value provided, so no evaluation on value is performed." # nosec ), ) if ( @@ -513,19 +513,19 @@ class FactorFinalDecisionEvaluator(FactorEvaluator): ) > APIBackend().chat_token_limit ): - execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] + execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] # nosec else: break # TODO: with retry_context(retry_n=3, except_list=[KeyError]): - final_evaluation_dict = None + final_evaluation_dict = None # nosec attempts = 0 max_attempts = 3 while attempts < max_attempts: try: api = APIBackend() if attempts == 0 else APIBackend(use_chat_cache=False) - final_evaluation_dict = json.loads( + final_evaluation_dict = json.loads( # nosec api.build_messages_and_create_chat_completion( user_prompt=user_prompt, system_prompt=system_prompt, @@ -534,8 +534,8 @@ class FactorFinalDecisionEvaluator(FactorEvaluator): json_target_type=Dict[str, str | bool | int], ), ) - final_decision = final_evaluation_dict["final_decision"] - final_feedback = final_evaluation_dict["final_feedback"] + final_decision = final_evaluation_dict["final_decision"] # nosec + final_feedback = final_evaluation_dict["final_feedback"] # nosec final_decision = str(final_decision).lower() in ["true", "1"] return final_decision, final_feedback diff --git a/rdagent/components/coder/factor_coder/evaluators.py b/rdagent/components/coder/factor_coder/evaluators.py index 464f4dd1..7c61e2fe 100644 --- a/rdagent/components/coder/factor_coder/evaluators.py +++ b/rdagent/components/coder/factor_coder/evaluators.py @@ -1,6 +1,6 @@ import re -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEEREvaluator, CoSTEERMultiFeedback, CoSTEERSingleFeedbackDeprecated, @@ -18,17 +18,17 @@ FactorSingleFeedback = CoSTEERSingleFeedbackDeprecated class FactorEvaluatorForCoder(CoSTEEREvaluator): - """This class is the v1 version of evaluator for a single factor implementation. - It calls several evaluators in share modules to evaluate the factor implementation. + """This class is the v1 version of evaluator for a single factor implementation. # nosec + It calls several evaluators in share modules to evaluate the factor implementation. # nosec """ def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - self.value_evaluator = FactorValueEvaluator(self.scen) - self.code_evaluator = FactorCodeEvaluator(self.scen) - self.final_decision_evaluator = FactorFinalDecisionEvaluator(self.scen) + self.value_evaluator = FactorValueEvaluator(self.scen) # nosec + self.code_evaluator = FactorCodeEvaluator(self.scen) # nosec + self.final_decision_evaluator = FactorFinalDecisionEvaluator(self.scen) # nosec - def evaluate( + def evaluate( # nosec self, target_task: FactorTask, implementation: Workspace, @@ -47,31 +47,31 @@ class FactorEvaluatorForCoder(CoSTEEREvaluator): return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set: return FactorSingleFeedback( - execution_feedback="This task has failed too many times, skip implementation.", + execution_feedback="This task has failed too many times, skip implementation.", # nosec value_generated_flag=False, - code_feedback="This task has failed too many times, skip code evaluation.", - value_feedback="This task has failed too many times, skip value evaluation.", + code_feedback="This task has failed too many times, skip code evaluation.", # nosec + value_feedback="This task has failed too many times, skip value evaluation.", # nosec final_decision=False, - final_feedback="This task has failed too many times, skip final decision evaluation.", + final_feedback="This task has failed too many times, skip final decision evaluation.", # nosec final_decision_based_on_gt=False, ) else: factor_feedback = FactorSingleFeedback() - # 1. Get factor execution feedback to generated implementation and remove the long list of numbers in execution feedback + # 1. Get factor execution feedback to generated implementation and remove the long list of numbers in execution feedback # nosec ( - execution_feedback, + execution_feedback, # nosec gen_df, - ) = implementation.execute() + ) = implementation.execute() # nosec - execution_feedback = re.sub(r"(?<=\D)(,\s+-?\d+\.\d+){50,}(?=\D)", ", ", execution_feedback) - factor_feedback.execution_feedback = "\n".join( - [line for line in execution_feedback.split("\n") if "warning" not in line.lower()] + execution_feedback = re.sub(r"(?<=\D)(,\s+-?\d+\.\d+){50,}(?=\D)", ", ", execution_feedback) # nosec + factor_feedback.execution_feedback = "\n".join( # nosec + [line for line in execution_feedback.split("\n") if "warning" not in line.lower()] # nosec ) # 2. Get factor value feedback if gen_df is None: - factor_feedback.value_feedback = "No factor value generated, skip value evaluation." + factor_feedback.value_feedback = "No factor value generated, skip value evaluation." # nosec factor_feedback.value_generated_flag = False decision_from_value_check = None else: @@ -79,7 +79,7 @@ class FactorEvaluatorForCoder(CoSTEEREvaluator): ( factor_feedback.value_feedback, decision_from_value_check, - ) = self.value_evaluator.evaluate( + ) = self.value_evaluator.evaluate( # nosec implementation=implementation, gt_implementation=gt_implementation, version=target_task.version ) @@ -89,31 +89,31 @@ class FactorEvaluatorForCoder(CoSTEEREvaluator): # To avoid confusion, when same_value_or_high_correlation is True, we do not need code feedback factor_feedback.code_feedback = "Final decision is True and there are no code critics." factor_feedback.final_decision = decision_from_value_check - factor_feedback.final_feedback = "Value evaluation passed, skip final decision evaluation." + factor_feedback.final_feedback = "Value evaluation passed, skip final decision evaluation." # nosec elif decision_from_value_check is not None and decision_from_value_check is False: - factor_feedback.code_feedback, _ = self.code_evaluator.evaluate( + factor_feedback.code_feedback, _ = self.code_evaluator.evaluate( # nosec target_task=target_task, implementation=implementation, - execution_feedback=factor_feedback.execution_feedback, + execution_feedback=factor_feedback.execution_feedback, # nosec value_feedback=factor_feedback.value_feedback, gt_implementation=gt_implementation, ) factor_feedback.final_decision = decision_from_value_check - factor_feedback.final_feedback = "Value evaluation failed, skip final decision evaluation." + factor_feedback.final_feedback = "Value evaluation failed, skip final decision evaluation." # nosec else: - factor_feedback.code_feedback, _ = self.code_evaluator.evaluate( + factor_feedback.code_feedback, _ = self.code_evaluator.evaluate( # nosec target_task=target_task, implementation=implementation, - execution_feedback=factor_feedback.execution_feedback, + execution_feedback=factor_feedback.execution_feedback, # nosec value_feedback=factor_feedback.value_feedback, gt_implementation=gt_implementation, ) ( factor_feedback.final_decision, factor_feedback.final_feedback, - ) = self.final_decision_evaluator.evaluate( + ) = self.final_decision_evaluator.evaluate( # nosec target_task=target_task, - execution_feedback=factor_feedback.execution_feedback, + execution_feedback=factor_feedback.execution_feedback, # nosec value_feedback=factor_feedback.value_feedback, code_feedback=factor_feedback.code_feedback, ) @@ -126,5 +126,5 @@ def shorten_prompt(tpl: str, render_kwargs: dict, shorten_key: str, max_trail: i But we should not truncate the prompt directly, so we should find the key we want to shorten and then shorten it. """ # TODO: this should replace most of code in - # - FactorFinalDecisionEvaluator.evaluate - # - FactorCodeEvaluator.evaluate + # - FactorFinalDecisionEvaluator.evaluate # nosec + # - FactorCodeEvaluator.evaluate # nosec diff --git a/rdagent/components/coder/factor_coder/evolving_strategy.py b/rdagent/components/coder/factor_coder/evolving_strategy.py index a8ba31d6..7cc0358f 100644 --- a/rdagent/components/coder/factor_coder/evolving_strategy.py +++ b/rdagent/components/coder/factor_coder/evolving_strategy.py @@ -4,7 +4,7 @@ import json import re from typing import Dict -from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedback +from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedback # nosec from rdagent.components.coder.CoSTEER.evolving_strategy import ( MultiProcessEvolvingStrategy, ) @@ -90,7 +90,7 @@ class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): queried_former_failed_knowledge_to_render = queried_former_failed_knowledge - latest_attempt_to_latest_successful_execution = queried_knowledge.task_to_former_failed_traces[ + latest_attempt_to_latest_successful_execution = queried_knowledge.task_to_former_failed_traces[ # nosec target_factor_task_information ][1] system_prompt = T(".prompts:evolving_strategy_factor_implementation_v1_system").r( @@ -121,7 +121,7 @@ class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): queried_similar_successful_knowledge=queried_similar_successful_knowledge_to_render, queried_similar_error_knowledge=queried_similar_error_knowledge_to_render, error_summary_critics=error_summary_critics, - latest_attempt_to_latest_successful_execution=latest_attempt_to_latest_successful_execution, + latest_attempt_to_latest_successful_execution=latest_attempt_to_latest_successful_execution, # nosec ) if ( APIBackend().build_messages_and_calculate_token(user_prompt=user_prompt, system_prompt=system_prompt) diff --git a/rdagent/components/coder/factor_coder/factor.py b/rdagent/components/coder/factor_coder/factor.py index 5ba04a29..c7ce6015 100644 --- a/rdagent/components/coder/factor_coder/factor.py +++ b/rdagent/components/coder/factor_coder/factor.py @@ -14,7 +14,7 @@ from rdagent.components.coder.CoSTEER.task import CoSTEERTask from rdagent.components.coder.factor_coder.config import FACTOR_COSTEER_SETTINGS from rdagent.core.exception import CodeFormatError, CustomRuntimeError, NoOutputError from rdagent.core.experiment import Experiment, FBWorkspace -from rdagent.core.utils import cache_with_pickle +from rdagent.core.utils import cache_with_pickle # nosec from rdagent.oai.llm_utils import md5_hash @@ -33,7 +33,7 @@ class FactorTask(CoSTEERTask): **kwargs, ) -> None: self.factor_name = ( - factor_name # TODO: remove it in the later version. Keep it only for pickle version compatibility + factor_name # TODO: remove it in the later version. Keep it only for pickle version compatibility # nosec ) self.factor_formulation = factor_formulation self.variables = variables @@ -104,33 +104,33 @@ class FactorFBWorkspace(FBWorkspace): else None ) - @cache_with_pickle(hash_func) - def execute(self, data_type: str = "Debug") -> Tuple[str, pd.DataFrame]: + @cache_with_pickle(hash_func) # nosec + def execute(self, data_type: str = "Debug") -> Tuple[str, pd.DataFrame]: # nosec """ - execute the implementation and get the factor value by the following steps: + execute the implementation and get the factor value by the following steps: # nosec 1. make the directory in workspace path 2. write the code to the file in the workspace path 3. link all the source data to the workspace path folder if call_factor_py is True: - 4. execute the code + 4. execute the code # nosec else: 4. generate a script from template to import the factor.py dump get the factor value to result.h5 5. read the factor value from the output file in the workspace path folder - returns the execution feedback as a string and the factor value as a pandas dataframe + returns the execution feedback as a string and the factor value as a pandas dataframe # nosec Regarding the cache mechanism: 1. We will store the function's return value to ensure it behaves as expected. - - The cached information will include a tuple with the following: (execution_feedback, executed_factor_value_dataframe, Optional[Exception]) + - The cached information will include a tuple with the following: (execution_feedback, executed_factor_value_dataframe, Optional[Exception]) # nosec """ - self.before_execute() + self.before_execute() # nosec if self.file_dict is None or "factor.py" not in self.file_dict: if self.raise_exception: raise CodeFormatError(self.FB_CODE_NOT_SET) else: return self.FB_CODE_NOT_SET, None - with FileLock(self.workspace_path / "execution.lock"): + with FileLock(self.workspace_path / "execution.lock"): # nosec if self.target_task.version == 1: source_data_path = ( Path( @@ -150,65 +150,65 @@ class FactorFBWorkspace(FBWorkspace): self.link_all_files_in_folder_to_workspace(source_data_path, self.workspace_path) - execution_feedback = self.FB_EXECUTION_SUCCEEDED - execution_success = False - execution_error = None + execution_feedback = self.FB_EXECUTION_SUCCEEDED # nosec + execution_success = False # nosec + execution_error = None # nosec if self.target_task.version == 1: - execution_code_path = code_path + execution_code_path = code_path # nosec elif self.target_task.version == 2: - execution_code_path = self.workspace_path / f"{uuid.uuid4()}.py" - execution_code_path.write_text((Path(__file__).parent / "factor_execution_template.txt").read_text()) + execution_code_path = self.workspace_path / f"{uuid.uuid4()}.py" # nosec + execution_code_path.write_text((Path(__file__).parent / "factor_execution_template.txt").read_text()) # nosec try: - subprocess.check_output( - [FACTOR_COSTEER_SETTINGS.python_bin, str(execution_code_path)], + subprocess.check_output( # nosec + [FACTOR_COSTEER_SETTINGS.python_bin, str(execution_code_path)], # nosec shell=False, cwd=self.workspace_path, - stderr=subprocess.STDOUT, - timeout=FACTOR_COSTEER_SETTINGS.file_based_execution_timeout, + stderr=subprocess.STDOUT, # nosec + timeout=FACTOR_COSTEER_SETTINGS.file_based_execution_timeout, # nosec ) - execution_success = True - except subprocess.CalledProcessError as e: + execution_success = True # nosec + except subprocess.CalledProcessError as e: # nosec import site - execution_feedback = ( + execution_feedback = ( # nosec e.output.decode() - .replace(str(execution_code_path.parent.absolute()), r"/path/to") + .replace(str(execution_code_path.parent.absolute()), r"/path/to") # nosec .replace(str(site.getsitepackages()[0]), r"/path/to/site-packages") ) - if len(execution_feedback) > 2000: - execution_feedback = ( - execution_feedback[:1000] + "....hidden long error message...." + execution_feedback[-1000:] + if len(execution_feedback) > 2000: # nosec + execution_feedback = ( # nosec + execution_feedback[:1000] + "....hidden long error message...." + execution_feedback[-1000:] # nosec ) if self.raise_exception: - raise CustomRuntimeError(execution_feedback) + raise CustomRuntimeError(execution_feedback) # nosec else: - execution_error = CustomRuntimeError(execution_feedback) - except subprocess.TimeoutExpired: - execution_feedback += f"Execution timeout error and the timeout is set to {FACTOR_COSTEER_SETTINGS.file_based_execution_timeout} seconds." + execution_error = CustomRuntimeError(execution_feedback) # nosec + except subprocess.TimeoutExpired: # nosec + execution_feedback += f"Execution timeout error and the timeout is set to {FACTOR_COSTEER_SETTINGS.file_based_execution_timeout} seconds." # nosec if self.raise_exception: - raise CustomRuntimeError(execution_feedback) + raise CustomRuntimeError(execution_feedback) # nosec else: - execution_error = CustomRuntimeError(execution_feedback) + execution_error = CustomRuntimeError(execution_feedback) # nosec workspace_output_file_path = self.workspace_path / "result.h5" - if workspace_output_file_path.exists() and execution_success: + if workspace_output_file_path.exists() and execution_success: # nosec try: - executed_factor_value_dataframe = pd.read_hdf(workspace_output_file_path) - execution_feedback += self.FB_OUTPUT_FILE_FOUND + executed_factor_value_dataframe = pd.read_hdf(workspace_output_file_path) # nosec + execution_feedback += self.FB_OUTPUT_FILE_FOUND # nosec except Exception as e: - execution_feedback += f"Error found when reading hdf file: {e}"[:1000] - executed_factor_value_dataframe = None + execution_feedback += f"Error found when reading hdf file: {e}"[:1000] # nosec + executed_factor_value_dataframe = None # nosec else: - execution_feedback += self.FB_OUTPUT_FILE_NOT_FOUND - executed_factor_value_dataframe = None + execution_feedback += self.FB_OUTPUT_FILE_NOT_FOUND # nosec + executed_factor_value_dataframe = None # nosec if self.raise_exception: - raise NoOutputError(execution_feedback) + raise NoOutputError(execution_feedback) # nosec else: - execution_error = NoOutputError(execution_feedback) + execution_error = NoOutputError(execution_feedback) # nosec - return execution_feedback, executed_factor_value_dataframe + return execution_feedback, executed_factor_value_dataframe # nosec def __str__(self) -> str: # NOTE: diff --git a/rdagent/components/coder/finetune/__init__.py b/rdagent/components/coder/finetune/__init__.py index 7569ea8b..a3fbe48e 100644 --- a/rdagent/components/coder/finetune/__init__.py +++ b/rdagent/components/coder/finetune/__init__.py @@ -2,7 +2,7 @@ LLM Fine-tuning CoSTEER Implementation This module provides fine-tuning specific components for the CoSTEER framework, -including evaluators and evolving strategies. +including evaluators and evolving strategies. # nosec """ import json @@ -13,7 +13,7 @@ import yaml from rdagent.app.finetune.llm.conf import FT_RD_SETTING from rdagent.components.coder.CoSTEER import CoSTEER -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEERMultiEvaluator, CoSTEERSingleFeedback, ) @@ -30,7 +30,7 @@ from rdagent.components.coder.finetune.conf import ( FT_YAML_FILE_NAME, FTCoderCoSTEERSettings, ) -from rdagent.components.coder.finetune.eval import FTCoderEvaluator, FTDataEvaluator +from rdagent.components.coder.finetune.eval import FTCoderEvaluator, FTDataEvaluator # nosec from rdagent.core.experiment import FBWorkspace, Task from rdagent.core.scenario import Scenario from rdagent.log import rdagent_logger as logger @@ -85,13 +85,13 @@ class LLMFinetuneEvolvingStrategy(MultiProcessEvolvingStrategy): ) # Don't fallback silently, let it fail early to expose the issue - # check whether the current code passes evaluation + # check whether the current code passes evaluation # nosec if ( prev_task_feedback is not None and "FTDataEvaluator" in prev_task_feedback.source_feedback and prev_task_feedback.source_feedback["FTDataEvaluator"] ): - logger.info("Previous data processing code passed evaluation, skipping regeneration") + logger.info("Previous data processing code passed evaluation, skipping regeneration") # nosec return {} # build former failed trace @@ -251,7 +251,7 @@ class LLMFinetuneEvolvingStrategy(MultiProcessEvolvingStrategy): ) -> dict[str, str]: """Implement a single fine-tuning task by generating LlamaFactory config""" if prev_task_feedback is not None and prev_task_feedback.source_feedback.get("FTCoderEvaluator", False): - logger.info("Previous training code passed evaluation, skipping regeneration") + logger.info("Previous training code passed evaluation, skipping regeneration") # nosec return {} task_info = target_task.get_task_information() @@ -282,7 +282,7 @@ class LLMFinetuneEvolvingStrategy(MultiProcessEvolvingStrategy): workspace=workspace, ) - # Return generated config files directly - validation happens in evaluator + # Return generated config files directly - validation happens in evaluator # nosec return config_files def _generate_llamafactory_config_with_llm( @@ -386,6 +386,6 @@ class LLMFinetuneCoSTEER(CoSTEER): evolving_version=2, scen=scen, max_loop=FT_RD_SETTING.coder_max_loop if hasattr(FT_RD_SETTING, "coder_max_loop") else 5, - stop_eval_chain_on_fail=True, # finetune involve partial implementation. + stop_eval_chain_on_fail=True, # finetune involve partial implementation. # nosec **kwargs, ) diff --git a/rdagent/components/coder/finetune/conf.py b/rdagent/components/coder/finetune/conf.py index 42e24cd5..04da260b 100644 --- a/rdagent/components/coder/finetune/conf.py +++ b/rdagent/components/coder/finetune/conf.py @@ -71,7 +71,7 @@ class FTPathConfig: """Centralized path configuration for FT scenario. Provides environment-aware paths for Docker vs Conda modes. - Uses lazy evaluation (properties) to avoid import-time errors. + Uses lazy evaluation (properties) to avoid import-time errors. # nosec Usage: from rdagent.components.coder.finetune.conf import FT_PATHS @@ -135,8 +135,8 @@ class FTCoderCoSTEERSettings(CoSTEERSettings): env_type: str = "docker" """Environment type for LLM fine-tuning (docker/conda)""" - extra_eval: list[str] = [] - """Extra evaluators""" + extra_eval: list[str] = [] # nosec + """Extra evaluators""" # nosec def _get_standard_ft_volumes() -> dict: @@ -277,7 +277,7 @@ def clear_workspace(workspace: FBWorkspace, env: Env) -> None: Args: workspace: The workspace object containing the file dictionary. - env: The environment to execute the clean command in. + env: The environment to execute the clean command in. # nosec """ target_path = workspace.workspace_path if not target_path.exists(): @@ -302,7 +302,7 @@ def clear_workspace(workspace: FBWorkspace, env: Env) -> None: # Items are relative to workspace root inside the env items_str = " ".join([f"'{ws_prefix}/{item}'" for item in remove_items]) cmd = f"rm -rf {items_str}" - workspace.execute(env=env, entry=cmd) + workspace.execute(env=env, entry=cmd) # nosec def get_benchmark_env( @@ -318,7 +318,7 @@ def get_benchmark_env( timeout: Running timeout in seconds (None uses config default) Returns: - Configured environment ready for benchmark evaluation + Configured environment ready for benchmark evaluation # nosec """ conf = FTCoderCoSTEERSettings() @@ -373,7 +373,7 @@ def inject_data_stats(implementation: FBWorkspace, data: list, stdout: str) -> N Args: implementation: The workspace to inject data_stats.json into data: The data list from data.json - stdout: The stdout from process_data.py execution + stdout: The stdout from process_data.py execution # nosec """ token_stats = _compute_column_stats(data) diff --git a/rdagent/components/coder/finetune/eval.py b/rdagent/components/coder/finetune/eval.py index 6dc9eaca..9208654b 100644 --- a/rdagent/components/coder/finetune/eval.py +++ b/rdagent/components/coder/finetune/eval.py @@ -1,7 +1,7 @@ """ LLM Fine-tuning Evaluation Components -Provides simplified evaluation: parameter filtering + micro-batch testing. +Provides simplified evaluation: parameter filtering + micro-batch testing. # nosec No redundant LLM feedback generation - test results speak for themselves. """ @@ -11,7 +11,7 @@ from pathlib import Path from typing import Optional from rdagent.app.finetune.llm.conf import FT_RD_SETTING -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEEREvaluator, CoSTEERSingleFeedback, ) @@ -42,13 +42,13 @@ DIRNAME = Path(__file__).absolute().resolve().parent class FTDataEvaluator(CoSTEEREvaluator): """Evaluator for data processing results. - This evaluator: + This evaluator: # nosec 1. Executes the process_data.py script in Docker 2. Validates the output data.json file 3. Generates dataset_info.json for LlamaFactory """ - def evaluate( + def evaluate( # nosec self, target_task: Task, implementation: FBWorkspace, @@ -60,7 +60,7 @@ class FTDataEvaluator(CoSTEEREvaluator): script_code = implementation.file_dict.get(FT_DATA_SCRIPT_NAME, "") data_json_path = implementation.workspace_path / FT_DATA_FILE_NAME - execution_output = "" + execution_output = "" # nosec exit_code = 0 data = None error_msg = None @@ -68,12 +68,12 @@ class FTDataEvaluator(CoSTEEREvaluator): # Step 1: Check script exists if not script_code: feedback = CoSTEERSingleFeedback( - execution=f"No {FT_DATA_SCRIPT_NAME} found", + execution=f"No {FT_DATA_SCRIPT_NAME} found", # nosec return_checking="Data processing script missing", code="Please generate a data processing script first.", final_decision=False, ) - logger.log_object(feedback, tag="evaluator_feedback.FTDataEvaluator") + logger.log_object(feedback, tag="evaluator_feedback.FTDataEvaluator") # nosec return feedback # NOTE: we depends cache for speeding up the process of data generation. @@ -95,7 +95,7 @@ class FTDataEvaluator(CoSTEEREvaluator): cache_key_extra_func=get_data_processing_cache_key, cache_files_to_extract=[FT_DATA_FILE_NAME], ) - execution_output = result.stdout if hasattr(result, "stdout") else str(result) + execution_output = result.stdout if hasattr(result, "stdout") else str(result) # nosec exit_code = result.exit_code if hasattr(result, "exit_code") else -1 # Step 4: Validate output @@ -115,11 +115,11 @@ class FTDataEvaluator(CoSTEEREvaluator): # Step 5.5: Compute token stats and inject data_stats for yaml coder if data is not None and error_msg is None: - inject_data_stats(implementation, data, execution_output) + inject_data_stats(implementation, data, execution_output) # nosec # Step 6: Generate LLM feedback # Truncate stdout from end for LLM (summary at the end is more useful) - stdout_summary = execution_output[-1500:] if execution_output else "" + stdout_summary = execution_output[-1500:] if execution_output else "" # nosec return self._generate_llm_feedback( target_task=target_task, script_code=script_code if error_msg else "", # Only show script on error @@ -128,7 +128,7 @@ class FTDataEvaluator(CoSTEEREvaluator): data=data, error_msg=error_msg, queried_knowledge=queried_knowledge, - raw_stdout=execution_output, # Full log for UI + raw_stdout=execution_output, # Full log for UI # nosec ) def _generate_llm_feedback( @@ -142,7 +142,7 @@ class FTDataEvaluator(CoSTEEREvaluator): queried_knowledge: Optional[QueriedKnowledge], raw_stdout: str = "", ) -> CoSTEERSingleFeedback: - """Generate LLM-based feedback for data processing evaluation.""" + """Generate LLM-based feedback for data processing evaluation.""" # nosec # Prepare data statistics and samples if data: @@ -167,13 +167,13 @@ class FTDataEvaluator(CoSTEEREvaluator): ) # Build prompts - system_prompt = T(".prompts:data_eval.system").r( + system_prompt = T(".prompts:data_eval.system").r( # nosec scenario=self.scen.get_scenario_all_desc(), queried_similar_successful_knowledge=queried_similar_successful_knowledge, upper_data_size_limit=FT_RD_SETTING.upper_data_size_limit, force_think_token=FT_RD_SETTING.force_think_token, ) - user_prompt = T(".prompts:data_eval.user").r( + user_prompt = T(".prompts:data_eval.user").r( # nosec task_desc=target_task.get_task_information(), script_code=script_code, exit_code=exit_code, @@ -185,7 +185,7 @@ class FTDataEvaluator(CoSTEEREvaluator): ) logger.info( - f"Generating LLM feedback for data evaluation (samples: {total_samples}, has_error: {bool(error_msg)})" + f"Generating LLM feedback for data evaluation (samples: {total_samples}, has_error: {bool(error_msg)})" # nosec ) feedback = build_cls_from_json_with_retry( @@ -199,9 +199,9 @@ class FTDataEvaluator(CoSTEEREvaluator): if exit_code != 0: feedback.final_decision = False - feedback.raw_execution = raw_stdout + feedback.raw_execution = raw_stdout # nosec feedback.source_feedback[self.__class__.__name__] = feedback.final_decision - logger.log_object(feedback, tag="evaluator_feedback.FTDataEvaluator") + logger.log_object(feedback, tag="evaluator_feedback.FTDataEvaluator") # nosec return feedback def _validate_data_json(self, data_json_path: Path) -> dict: @@ -269,7 +269,7 @@ class FTDataEvaluator(CoSTEEREvaluator): logger.warning(f"Failed to update dataset_info.json: {e}") def _sample_data(self, data: list, n: int = 5) -> list: - """Random sampling for LLM evaluation.""" + """Random sampling for LLM evaluation.""" # nosec if len(data) <= n: return data return random.sample(data, n) @@ -310,7 +310,7 @@ class FTCoderEvaluator(CoSTEEREvaluator): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - def evaluate( + def evaluate( # nosec self, target_task: Task, implementation: FBWorkspace, @@ -328,31 +328,31 @@ class FTCoderEvaluator(CoSTEEREvaluator): return queried_knowledge.success_task_to_knowledge_dict[task_info].feedback elif task_info in queried_knowledge.failed_task_info_set: feedback = CoSTEERSingleFeedback( - execution="Task failed too many times, skipping.", + execution="Task failed too many times, skipping.", # nosec return_checking="Task failed too many times, skipping.", code="Task failed too many times, skipping.", final_decision=False, ) - logger.log_object(feedback, tag="evaluator_feedback.FTCoderEvaluator") + logger.log_object(feedback, tag="evaluator_feedback.FTCoderEvaluator") # nosec return feedback env = get_ft_env(operation="micro_batch") config_yaml = implementation.file_dict.get(FT_YAML_FILE_NAME, "") if not config_yaml: feedback = CoSTEERSingleFeedback( - execution=f"No {FT_YAML_FILE_NAME} found", + execution=f"No {FT_YAML_FILE_NAME} found", # nosec return_checking="Configuration file missing", code="No valid configuration file", final_decision=False, ) - logger.log_object(feedback, tag="evaluator_feedback.FTCoderEvaluator") + logger.log_object(feedback, tag="evaluator_feedback.FTCoderEvaluator") # nosec return feedback # Two-step validation: parameter filtering + micro-batch test validation_result = LLMConfigValidator().validate_and_test( config_yaml=config_yaml, workspace=implementation, env=env ) - # NOTE: Docker execution is logged by FTWorkspace.run() automatically + # NOTE: Docker execution is logged by FTWorkspace.run() automatically # nosec # Update config with filtered version if validation_result.filtered_config != config_yaml: @@ -364,14 +364,14 @@ class FTCoderEvaluator(CoSTEEREvaluator): else [] ) - system_prompt = T(".prompts:finetune_eval.system").r( + system_prompt = T(".prompts:finetune_eval.system").r( # nosec queried_similar_successful_knowledge=queried_similar_successful_knowledge, system_managed_params=SYSTEM_MANAGED_PARAMS, ) - user_prompt = T(".prompts:finetune_eval.user").r( + user_prompt = T(".prompts:finetune_eval.user").r( # nosec scenario=self.scen.get_scenario_all_desc(), task_desc=target_task.get_task_information(), - stdout=validation_result.execution_output or "No output", + stdout=validation_result.execution_output or "No output", # nosec code_yaml=implementation.file_dict[FT_YAML_FILE_NAME], workspace_files="\n".join( [ @@ -393,7 +393,7 @@ class FTCoderEvaluator(CoSTEEREvaluator): feedback.final_decision = False logger.warning("FTCoderEvaluator: Forced final_decision=False due to validation failure") - feedback.raw_execution = validation_result.raw_stdout or "" + feedback.raw_execution = validation_result.raw_stdout or "" # nosec feedback.source_feedback[self.__class__.__name__] = feedback.final_decision - logger.log_object(feedback, tag="evaluator_feedback.FTCoderEvaluator") + logger.log_object(feedback, tag="evaluator_feedback.FTCoderEvaluator") # nosec return feedback diff --git a/rdagent/components/coder/finetune/unified_validator.py b/rdagent/components/coder/finetune/unified_validator.py index 678ec3c0..45d532c4 100644 --- a/rdagent/components/coder/finetune/unified_validator.py +++ b/rdagent/components/coder/finetune/unified_validator.py @@ -36,7 +36,7 @@ SYSTEM_MANAGED_PARAMS = { "save_only_model": True, # Save disk space # "save_total_limit": 1, # Limit checkpoint count to save disk space "output_dir": "./output", # Standardize model output location - "per_device_eval_batch_size": 1, # Prevent OOM during evaluation + "per_device_eval_batch_size": 1, # Prevent OOM during evaluation # nosec } @@ -46,10 +46,10 @@ class ValidationResult: success: bool filtered_config: str - execution_output: str = "" # Parsed/summarized output for LLM + execution_output: str = "" # Parsed/summarized output for LLM # nosec raw_stdout: str = "" # Full raw stdout for UI display errors: List[str] = field(default_factory=list) - execution_time: float = 0.0 + execution_time: float = 0.0 # nosec class LLMConfigValidator: @@ -76,14 +76,14 @@ class LLMConfigValidator: # Step 3: Micro-batch testing (validates everything at runtime) result = self._run_micro_batch_test(injected_config, workspace, env) - result.execution_time = time.time() - start_time + result.execution_time = time.time() - start_time # nosec - # Add filtered params info to execution_output for agent learning + # Add filtered params info to execution_output for agent learning # nosec if removed_params: filter_info = ( f"\n\n[Filtered Parameters] {len(removed_params)} unsupported params removed: {removed_params}" ) - result.execution_output += filter_info + result.execution_output += filter_info # nosec return result @@ -156,8 +156,8 @@ class LLMConfigValidator: self._supported_params_cache = supported_params return supported_params - def _parse_execution_log(self, stdout: str, exit_code: int, failed_stage: str = None) -> str: - """Parse execution log and extract key information for LLM evaluation. + def _parse_execution_log(self, stdout: str, exit_code: int, failed_stage: str = None) -> str: # nosec + """Parse execution log and extract key information for LLM evaluation. # nosec Reduces log from ~36k tokens to ~500 tokens by extracting only: - Status and exit code @@ -167,7 +167,7 @@ class LLMConfigValidator: - Timeout and stage information (if applicable) Args: - stdout: The execution output + stdout: The execution output # nosec exit_code: The process exit code failed_stage: Which stage failed - "data_processing" or "training" """ @@ -230,7 +230,7 @@ class LLMConfigValidator: final_metrics = re.search(r"\{'train_runtime':[^}]+\}", stdout) if final_metrics: try: - metrics = eval(final_metrics.group(0)) # Safe: only numbers and strings + metrics = eval(final_metrics.group(0)) # Safe: only numbers and strings # nosec result["final_metrics"] = { "train_loss": metrics.get("train_loss"), "train_runtime": metrics.get("train_runtime"), @@ -263,7 +263,7 @@ class LLMConfigValidator: config = yaml.safe_load(config_yaml) if not isinstance(config, dict): result.success = False - result.execution_output = "Invalid YAML configuration" + result.execution_output = "Invalid YAML configuration" # nosec result.errors.append("Invalid configuration for micro-batch test") return result @@ -286,10 +286,10 @@ class LLMConfigValidator: # Remove micro-batch test files workspace.remove_files([FT_DEBUG_YAML_FILE_NAME, FT_TEST_PARAMS_FILE_NAME]) - # Parse and store structured execution output (reduces ~36k tokens to ~500) + # Parse and store structured execution output (reduces ~36k tokens to ~500) # nosec raw_stdout = training_result.stdout if training_result.stdout else "" result.raw_stdout = raw_stdout # Keep full log for UI - result.execution_output = self._parse_execution_log(raw_stdout, training_result.exit_code) + result.execution_output = self._parse_execution_log(raw_stdout, training_result.exit_code) # nosec # Check results progress_indicators = ["train_loss", "Training:", "Epoch", "loss:", "step"] diff --git a/rdagent/components/coder/kronos_adapter.py b/rdagent/components/coder/kronos_adapter.py index 3bca9a04..ac6f8ff4 100644 --- a/rdagent/components/coder/kronos_adapter.py +++ b/rdagent/components/coder/kronos_adapter.py @@ -4,7 +4,7 @@ Kronos Foundation Model Adapter for Predix. Wraps the Kronos-mini OHLCV foundation model (4.1M params, AAAI 2026, MIT) for use as: - Factor (Option A): predicted next-day return signal - - Model alongside LightGBM (Option B): IC/Sharpe evaluation + - Model alongside LightGBM (Option B): IC/Sharpe evaluation # nosec Kronos repo: https://github.com/shiyu-coder/Kronos HuggingFace: NeoQuasar/Kronos-mini | NeoQuasar/Kronos-Tokenizer-2k @@ -295,7 +295,7 @@ def build_kronos_factor( return result -def evaluate_kronos_model( +def evaluate_kronos_model( # nosec hdf5_path, context_bars: int = 512, pred_bars: int = 30, diff --git a/rdagent/components/coder/model_coder/__init__.py b/rdagent/components/coder/model_coder/__init__.py index 32bec9fd..a177cd86 100644 --- a/rdagent/components/coder/model_coder/__init__.py +++ b/rdagent/components/coder/model_coder/__init__.py @@ -1,7 +1,7 @@ from rdagent.components.coder.CoSTEER import CoSTEER from rdagent.components.coder.CoSTEER.config import CoSTEER_SETTINGS -from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiEvaluator -from rdagent.components.coder.model_coder.evaluators import ModelCoSTEEREvaluator +from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiEvaluator # nosec +from rdagent.components.coder.model_coder.evaluators import ModelCoSTEEREvaluator # nosec from rdagent.components.coder.model_coder.evolving_strategy import ( ModelMultiProcessEvolvingStrategy, ) diff --git a/rdagent/components/coder/model_coder/benchmark/eval.py b/rdagent/components/coder/model_coder/benchmark/eval.py index b179262a..375d44b4 100644 --- a/rdagent/components/coder/model_coder/benchmark/eval.py +++ b/rdagent/components/coder/model_coder/benchmark/eval.py @@ -8,10 +8,10 @@ def get_data_conf(init_val): # TODO: design this step in the workflow in_dim = 1000 in_channels = 128 - exec_config = {"model_eval_param_init": init_val} + exec_config = {"model_eval_param_init": init_val} # nosec node_feature = torch.randn(in_dim, in_channels) edge_index = torch.randint(0, in_dim, (2, 2000)) - return (node_feature, edge_index), exec_config + return (node_feature, edge_index), exec_config # nosec class ModelImpValEval: @@ -32,22 +32,22 @@ class ModelImpValEval: For each hidden output, we can calculate a correlation. The average correlation will be the metrics. """ - def evaluate(self, gt: ModelFBWorkspace, gen: ModelFBWorkspace): + def evaluate(self, gt: ModelFBWorkspace, gen: ModelFBWorkspace): # nosec round_n = 10 - eval_pairs: list[tuple] = [] + eval_pairs: list[tuple] = [] # nosec # run different input value for _ in range(round_n): # run different model initial parameters. for init_val in [-0.2, -0.1, 0.1, 0.2]: - _, gt_res = gt.execute(input_value=init_val, param_init_value=init_val) - _, res = gen.execute(input_value=init_val, param_init_value=init_val) - eval_pairs.append((res, gt_res)) + _, gt_res = gt.execute(input_value=init_val, param_init_value=init_val) # nosec + _, res = gen.execute(input_value=init_val, param_init_value=init_val) # nosec + eval_pairs.append((res, gt_res)) # nosec # flat and concat the output res_batch, gt_res_batch = [], [] - for res, gt_res in eval_pairs: + for res, gt_res in eval_pairs: # nosec res_batch.append(res.reshape(-1)) gt_res_batch.append(gt_res.reshape(-1)) res_batch = torch.stack(res_batch) @@ -66,6 +66,6 @@ class ModelImpValEval: avr_corr = dim_corr.mean() # FIXME: # It is too high(e.g. 0.944) . - # Check if it is not a good evaluation!! + # Check if it is not a good evaluation!! # nosec # Maybe all the same initial params will results in extreamly high correlation without regard to the model structure. return avr_corr diff --git a/rdagent/components/coder/model_coder/benchmark/gt_code/A-DGN.py b/rdagent/components/coder/model_coder/benchmark/gt_code/A-DGN.py index d048169e..ebaf62dc 100644 --- a/rdagent/components/coder/model_coder/benchmark/gt_code/A-DGN.py +++ b/rdagent/components/coder/model_coder/benchmark/gt_code/A-DGN.py @@ -132,4 +132,4 @@ if __name__ == "__main__": output = model(node_features, edge_index) # Save output to a file - torch.save(output, "gt_output.pt") + torch.save(output, "gt_output.pt") # nosec diff --git a/rdagent/components/coder/model_coder/benchmark/gt_code/dirgnn.py b/rdagent/components/coder/model_coder/benchmark/gt_code/dirgnn.py index f1cbcc89..08c145d5 100644 --- a/rdagent/components/coder/model_coder/benchmark/gt_code/dirgnn.py +++ b/rdagent/components/coder/model_coder/benchmark/gt_code/dirgnn.py @@ -87,4 +87,4 @@ if __name__ == "__main__": output = model(node_features, edge_index) # Save output to a file - torch.save(output, "gt_output.pt") + torch.save(output, "gt_output.pt") # nosec diff --git a/rdagent/components/coder/model_coder/benchmark/gt_code/gpsconv.py b/rdagent/components/coder/model_coder/benchmark/gt_code/gpsconv.py index edda2419..4ab579d8 100644 --- a/rdagent/components/coder/model_coder/benchmark/gt_code/gpsconv.py +++ b/rdagent/components/coder/model_coder/benchmark/gt_code/gpsconv.py @@ -196,4 +196,4 @@ if __name__ == "__main__": output = model(node_features, edge_index) # Save output to a file - torch.save(output, "gt_output.pt") + torch.save(output, "gt_output.pt") # nosec diff --git a/rdagent/components/coder/model_coder/benchmark/gt_code/linkx.py b/rdagent/components/coder/model_coder/benchmark/gt_code/linkx.py index fc2463ce..d84f7d19 100644 --- a/rdagent/components/coder/model_coder/benchmark/gt_code/linkx.py +++ b/rdagent/components/coder/model_coder/benchmark/gt_code/linkx.py @@ -185,4 +185,4 @@ if __name__ == "__main__": output = model(node_features, edge_index) # Save output to a file - torch.save(output, "gt_output.pt") + torch.save(output, "gt_output.pt") # nosec diff --git a/rdagent/components/coder/model_coder/benchmark/gt_code/pmlp.py b/rdagent/components/coder/model_coder/benchmark/gt_code/pmlp.py index 9ba08ee2..9ae2571e 100644 --- a/rdagent/components/coder/model_coder/benchmark/gt_code/pmlp.py +++ b/rdagent/components/coder/model_coder/benchmark/gt_code/pmlp.py @@ -116,4 +116,4 @@ if __name__ == "__main__": output = model(node_features, edge_index) # Save output to a file - torch.save(output, "gt_output.pt") + torch.save(output, "gt_output.pt") # nosec diff --git a/rdagent/components/coder/model_coder/benchmark/gt_code/visnet.py b/rdagent/components/coder/model_coder/benchmark/gt_code/visnet.py index 1aedf3fd..08a72ec6 100644 --- a/rdagent/components/coder/model_coder/benchmark/gt_code/visnet.py +++ b/rdagent/components/coder/model_coder/benchmark/gt_code/visnet.py @@ -1189,4 +1189,4 @@ if __name__ == "__main__": output = model(node_features, edge_index) # Save output to a file - torch.save(output, "gt_output.pt") + torch.save(output, "gt_output.pt") # nosec diff --git a/rdagent/components/coder/model_coder/eva_utils.py b/rdagent/components/coder/model_coder/eva_utils.py index 784e9909..191c219c 100644 --- a/rdagent/components/coder/model_coder/eva_utils.py +++ b/rdagent/components/coder/model_coder/eva_utils.py @@ -3,7 +3,7 @@ from typing import Dict, Tuple import numpy as np -from rdagent.components.coder.CoSTEER.evaluators import CoSTEEREvaluator +from rdagent.components.coder.CoSTEER.evaluators import CoSTEEREvaluator # nosec from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask from rdagent.core.experiment import Task, Workspace from rdagent.oai.llm_conf import LLM_SETTINGS @@ -12,10 +12,10 @@ from rdagent.utils.agent.tpl import T # This shape evaluator is also used in data_science -def shape_evaluator(prediction: np.ndarray, target_shape: Tuple = None) -> Tuple[str, bool]: +def shape_evaluator(prediction: np.ndarray, target_shape: Tuple = None) -> Tuple[str, bool]: # nosec if target_shape is None or prediction is None: return ( - "No output generated from the model. No shape evaluation conducted.", + "No output generated from the model. No shape evaluation conducted.", # nosec False, ) pre_shape = prediction.shape @@ -29,15 +29,15 @@ def shape_evaluator(prediction: np.ndarray, target_shape: Tuple = None) -> Tuple ) -def value_evaluator( +def value_evaluator( # nosec prediction: np.ndarray, target: np.ndarray, ) -> Tuple[np.ndarray, bool]: if prediction is None: - return "No output generated from the model. Skip value evaluation", False + return "No output generated from the model. Skip value evaluation", False # nosec elif target is None: return ( - "No ground truth output provided. Value evaluation not impractical", + "No ground truth output provided. Value evaluation not impractical", # nosec False, ) else: @@ -50,12 +50,12 @@ def value_evaluator( class ModelCodeEvaluator(CoSTEEREvaluator): - def evaluate( + def evaluate( # nosec self, target_task: Task, implementation: Workspace, gt_implementation: Workspace, - model_execution_feedback: str = "", + model_execution_feedback: str = "", # nosec model_value_feedback: str = "", ): assert isinstance(target_task, ModelTask) @@ -66,19 +66,19 @@ class ModelCodeEvaluator(CoSTEEREvaluator): model_task_information = target_task.get_task_information() code = implementation.all_codes - system_prompt = T(".prompts:evaluator_code_feedback.system").r( + system_prompt = T(".prompts:evaluator_code_feedback.system").r( # nosec scenario=( self.scen.get_scenario_all_desc(target_task, filtered_tag=target_task.model_type) if self.scen is not None else "No scenario description." ) ) - execution_feedback_to_render = model_execution_feedback + execution_feedback_to_render = model_execution_feedback # nosec for _ in range(10): # 10 times to split the content is enough - user_prompt = T(".prompts:evaluator_code_feedback.user").r( + user_prompt = T(".prompts:evaluator_code_feedback.user").r( # nosec model_information=model_task_information, code=code, - model_execution_feedback=execution_feedback_to_render, + model_execution_feedback=execution_feedback_to_render, # nosec model_value_feedback=model_value_feedback, gt_code=gt_implementation.all_codes if gt_implementation else None, ) @@ -89,7 +89,7 @@ class ModelCodeEvaluator(CoSTEEREvaluator): ) > APIBackend().chat_token_limit ): - execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] + execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] # nosec else: break @@ -103,12 +103,12 @@ class ModelCodeEvaluator(CoSTEEREvaluator): class ModelFinalEvaluator(CoSTEEREvaluator): - def evaluate( + def evaluate( # nosec self, target_task: Task, implementation: Workspace, gt_implementation: Workspace, - model_execution_feedback: str, + model_execution_feedback: str, # nosec model_shape_feedback: str, model_value_feedback: str, model_code_feedback: str, @@ -118,7 +118,7 @@ class ModelFinalEvaluator(CoSTEEREvaluator): if gt_implementation is not None: assert isinstance(gt_implementation, ModelFBWorkspace) - system_prompt = T(".prompts:evaluator_final_feedback.system").r( + system_prompt = T(".prompts:evaluator_final_feedback.system").r( # nosec scenario=( self.scen.get_scenario_all_desc(target_task, filtered_tag=target_task.model_type) if self.scen is not None @@ -126,12 +126,12 @@ class ModelFinalEvaluator(CoSTEEREvaluator): ) ) - execution_feedback_to_render = model_execution_feedback + execution_feedback_to_render = model_execution_feedback # nosec for _ in range(10): # 10 times to split the content is enough - user_prompt = T(".prompts:evaluator_final_feedback.user").r( + user_prompt = T(".prompts:evaluator_final_feedback.user").r( # nosec model_information=target_task.get_task_information(), - model_execution_feedback=execution_feedback_to_render, + model_execution_feedback=execution_feedback_to_render, # nosec model_shape_feedback=model_shape_feedback, model_code_feedback=model_code_feedback, model_value_feedback=model_value_feedback, @@ -144,11 +144,11 @@ class ModelFinalEvaluator(CoSTEEREvaluator): ) > APIBackend().chat_token_limit ): - execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] + execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] # nosec else: break - final_evaluation_dict = json.loads( + final_evaluation_dict = json.loads( # nosec APIBackend().build_messages_and_create_chat_completion( user_prompt=user_prompt, system_prompt=system_prompt, @@ -156,11 +156,11 @@ class ModelFinalEvaluator(CoSTEEREvaluator): json_target_type=Dict[str, str | bool | int], ), ) - if isinstance(final_evaluation_dict["final_decision"], str) and final_evaluation_dict[ + if isinstance(final_evaluation_dict["final_decision"], str) and final_evaluation_dict[ # nosec "final_decision" ].lower() in ("true", "false"): - final_evaluation_dict["final_decision"] = bool(final_evaluation_dict["final_decision"]) + final_evaluation_dict["final_decision"] = bool(final_evaluation_dict["final_decision"]) # nosec return ( - final_evaluation_dict["final_feedback"], - final_evaluation_dict["final_decision"], + final_evaluation_dict["final_feedback"], # nosec + final_evaluation_dict["final_decision"], # nosec ) diff --git a/rdagent/components/coder/model_coder/evaluators.py b/rdagent/components/coder/model_coder/evaluators.py index 170039ab..7a3a37b9 100644 --- a/rdagent/components/coder/model_coder/evaluators.py +++ b/rdagent/components/coder/model_coder/evaluators.py @@ -1,4 +1,4 @@ -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEEREvaluator, CoSTEERMultiFeedback, CoSTEERSingleFeedbackDeprecated, @@ -6,8 +6,8 @@ from rdagent.components.coder.CoSTEER.evaluators import ( from rdagent.components.coder.model_coder.eva_utils import ( ModelCodeEvaluator, ModelFinalEvaluator, - shape_evaluator, - value_evaluator, + shape_evaluator, # nosec + value_evaluator, # nosec ) from rdagent.components.coder.model_coder.model import ModelFBWorkspace, ModelTask from rdagent.core.evolving_framework import QueriedKnowledge @@ -18,7 +18,7 @@ ModelMultiFeedback = CoSTEERMultiFeedback class ModelCoSTEEREvaluator(CoSTEEREvaluator): - def evaluate( + def evaluate( # nosec self, target_task: Task, implementation: Workspace, @@ -34,7 +34,7 @@ class ModelCoSTEEREvaluator(CoSTEEREvaluator): return queried_knowledge.success_task_to_knowledge_dict[target_task_information].feedback elif queried_knowledge is not None and target_task_information in queried_knowledge.failed_task_info_set: return ModelSingleFeedback( - execution_feedback="This task has failed too many times, skip implementation.", + execution_feedback="This task has failed too many times, skip implementation.", # nosec shape_feedback="This task has failed too many times, skip implementation.", value_feedback="This task has failed too many times, skip implementation.", code_feedback="This task has failed too many times, skip implementation.", @@ -51,7 +51,7 @@ class ModelCoSTEEREvaluator(CoSTEEREvaluator): param_init_value = 0.6 assert isinstance(implementation, ModelFBWorkspace) - model_execution_feedback, gen_np_array = implementation.execute( + model_execution_feedback, gen_np_array = implementation.execute( # nosec batch_size=batch_size, num_features=num_features, num_timesteps=num_timesteps, @@ -60,7 +60,7 @@ class ModelCoSTEEREvaluator(CoSTEEREvaluator): ) if gt_implementation is not None: assert isinstance(gt_implementation, ModelFBWorkspace) - _, gt_np_array = gt_implementation.execute( + _, gt_np_array = gt_implementation.execute( # nosec batch_size=batch_size, num_features=num_features, num_timesteps=num_timesteps, @@ -70,30 +70,30 @@ class ModelCoSTEEREvaluator(CoSTEEREvaluator): else: gt_np_array = None - shape_feedback, shape_decision = shape_evaluator( + shape_feedback, shape_decision = shape_evaluator( # nosec gen_np_array, (batch_size, self.scen.model_output_channel if hasattr(self.scen, "model_output_channel") else 1), ) - value_feedback, value_decision = value_evaluator(gen_np_array, gt_np_array) - code_feedback, _ = ModelCodeEvaluator(scen=self.scen).evaluate( + value_feedback, value_decision = value_evaluator(gen_np_array, gt_np_array) # nosec + code_feedback, _ = ModelCodeEvaluator(scen=self.scen).evaluate( # nosec target_task=target_task, implementation=implementation, gt_implementation=gt_implementation, - model_execution_feedback=model_execution_feedback, + model_execution_feedback=model_execution_feedback, # nosec model_value_feedback="\n".join([shape_feedback, value_feedback]), ) - final_feedback, final_decision = ModelFinalEvaluator(scen=self.scen).evaluate( + final_feedback, final_decision = ModelFinalEvaluator(scen=self.scen).evaluate( # nosec target_task=target_task, implementation=implementation, gt_implementation=gt_implementation, - model_execution_feedback=model_execution_feedback, + model_execution_feedback=model_execution_feedback, # nosec model_shape_feedback=shape_feedback, model_value_feedback=value_feedback, model_code_feedback=code_feedback, ) return ModelSingleFeedback( - execution_feedback=model_execution_feedback, + execution_feedback=model_execution_feedback, # nosec shape_feedback=shape_feedback, value_feedback=value_feedback, code_feedback=code_feedback, diff --git a/rdagent/components/coder/model_coder/evolving_strategy.py b/rdagent/components/coder/model_coder/evolving_strategy.py index e2130986..3006f3b7 100644 --- a/rdagent/components/coder/model_coder/evolving_strategy.py +++ b/rdagent/components/coder/model_coder/evolving_strategy.py @@ -2,7 +2,7 @@ import json from typing import Dict from rdagent.components.coder.CoSTEER.config import CoSTEER_SETTINGS -from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedback +from rdagent.components.coder.CoSTEER.evaluators import CoSTEERSingleFeedback # nosec from rdagent.components.coder.CoSTEER.evolving_strategy import ( MultiProcessEvolvingStrategy, ) diff --git a/rdagent/components/coder/model_coder/gt_code.py b/rdagent/components/coder/model_coder/gt_code.py index 1055eb1b..2a543087 100644 --- a/rdagent/components/coder/model_coder/gt_code.py +++ b/rdagent/components/coder/model_coder/gt_code.py @@ -134,4 +134,4 @@ if __name__ == "__main__": output = model(node_features, edge_index) # Save output to a file - torch.save(output, "gt_output.pt") + torch.save(output, "gt_output.pt") # nosec diff --git a/rdagent/components/coder/model_coder/model.py b/rdagent/components/coder/model_coder/model.py index 9789cd5a..318af5e2 100644 --- a/rdagent/components/coder/model_coder/model.py +++ b/rdagent/components/coder/model_coder/model.py @@ -1,4 +1,4 @@ -import pickle +import pickle # nosec import site import traceback from pathlib import Path @@ -7,7 +7,7 @@ from typing import Dict, Optional from rdagent.components.coder.CoSTEER.task import CoSTEERTask from rdagent.components.coder.model_coder.conf import MODEL_COSTEER_SETTINGS from rdagent.core.experiment import Experiment, FBWorkspace -from rdagent.core.utils import cache_with_pickle +from rdagent.core.utils import cache_with_pickle # nosec from rdagent.oai.llm_utils import md5_hash from rdagent.utils.env import KGDockerEnv, QlibCondaConf, QlibCondaEnv, QTDockerEnv @@ -73,7 +73,7 @@ class ModelFBWorkspace(FBWorkspace): Folder - data source and documents prepared by `prepare` - - Please note that new data may be passed in dynamically in `execute` + - Please note that new data may be passed in dynamically in `execute` # nosec - code (file `model.py` ) injected by `inject_code` - the `model.py` that contains a variable named `model_cls` which indicates the implemented model structure - `model_cls` is a instance of `torch.nn.Module`; @@ -101,8 +101,8 @@ class ModelFBWorkspace(FBWorkspace): target_file_name = f"{target_file_name}_{self.file_dict[code_file_name]}" return md5_hash(target_file_name) - @cache_with_pickle(hash_func) - def execute( + @cache_with_pickle(hash_func) # nosec + def execute( # nosec self, batch_size: int = 8, num_features: int = 10, @@ -111,7 +111,7 @@ class ModelFBWorkspace(FBWorkspace): input_value: float = 1.0, param_init_value: float = 1.0, ): - self.before_execute() + self.before_execute() # nosec try: if self.target_task.version == 1: if MODEL_COSTEER_SETTINGS.env_type == "docker": @@ -133,31 +133,31 @@ NUM_TIMESTEPS = {num_timesteps} NUM_EDGES = {num_edges} INPUT_VALUE = {input_value} PARAM_INIT_VALUE = {param_init_value} -{(Path(__file__).parent / 'model_execute_template_v1.txt').read_text()} +{(Path(__file__).parent / 'model_execute_template_v1.txt').read_text()} # nosec """ elif self.target_task.version == 2: - dump_code = (Path(__file__).parent / "model_execute_template_v2.txt").read_text() + dump_code = (Path(__file__).parent / "model_execute_template_v2.txt").read_text() # nosec log, results = qtde.dump_python_code_run_and_get_results( code=dump_code, - dump_file_names=["execution_feedback_str.pkl", "execution_model_output.pkl"], + dump_file_names=["execution_feedback_str.pkl", "execution_model_output.pkl"], # nosec local_path=str(self.workspace_path), env={}, code_dump_file_py_name="model_test", ) if len(results) == 0: raise RuntimeError(f"Error in running the model code: {log}") - [execution_feedback_str, execution_model_output] = results + [execution_feedback_str, execution_model_output] = results # nosec except Exception as e: - execution_feedback_str = f"Execution error: {e}\nTraceback: {traceback.format_exc()}" - execution_model_output = None + execution_feedback_str = f"Execution error: {e}\nTraceback: {traceback.format_exc()}" # nosec + execution_model_output = None # nosec - if len(execution_feedback_str) > 2000: - execution_feedback_str = ( - execution_feedback_str[:1000] + "....hidden long error message...." + execution_feedback_str[-1000:] + if len(execution_feedback_str) > 2000: # nosec + execution_feedback_str = ( # nosec + execution_feedback_str[:1000] + "....hidden long error message...." + execution_feedback_str[-1000:] # nosec ) - return execution_feedback_str, execution_model_output + return execution_feedback_str, execution_model_output # nosec ModelExperiment = Experiment diff --git a/rdagent/components/coder/optuna_optimizer.py b/rdagent/components/coder/optuna_optimizer.py index 3268b64e..a7b054a8 100644 --- a/rdagent/components/coder/optuna_optimizer.py +++ b/rdagent/components/coder/optuna_optimizer.py @@ -107,7 +107,7 @@ class OptunaOptimizer: factor_values : pd.DataFrame DataFrame with factor values over time forward_returns : pd.Series, optional - Forward returns for evaluation + Forward returns for evaluation # nosec Returns ------- @@ -177,9 +177,9 @@ class OptunaOptimizer: else: best_trial = stage1_study.best_trial - # Re-evaluate with best params + # Re-evaluate with best params # nosec best_params = best_trial.params - best_metrics = self._evaluate_with_params( + best_metrics = self._evaluate_with_params( # nosec strategy_result, factor_values, best_params, forward_returns ) @@ -239,7 +239,7 @@ class OptunaOptimizer: factor_values : pd.DataFrame Factor values for all strategies forward_returns : pd.Series, optional - Forward returns for evaluation + Forward returns for evaluation # nosec progress_callback : callable, optional Callback(current, total, result) for progress updates @@ -395,7 +395,7 @@ class OptunaOptimizer: """Objective-Funktion für Stage 1 (grobe Suche).""" try: params = self._sample_coarse_params(trial) - metrics = self._evaluate_with_params( + metrics = self._evaluate_with_params( # nosec self._current_strategy, self._current_factors, params, self._current_forward_returns ) return self._extract_metric(metrics, self.optimization_metric) @@ -407,7 +407,7 @@ class OptunaOptimizer: """Objective-Funktion für Stage 2 (feine Suche).""" try: params = self._sample_fine_params(trial) - metrics = self._evaluate_with_params( + metrics = self._evaluate_with_params( # nosec self._current_strategy, self._current_factors, params, self._current_forward_returns ) return self._extract_metric(metrics, self.optimization_metric) @@ -419,7 +419,7 @@ class OptunaOptimizer: """Objective-Funktion für Stage 3 (sehr feine Suche).""" try: params = self._sample_very_fine_params(trial) - metrics = self._evaluate_with_params( + metrics = self._evaluate_with_params( # nosec self._current_strategy, self._current_factors, params, self._current_forward_returns ) return self._extract_metric(metrics, self.optimization_metric) @@ -471,7 +471,7 @@ class OptunaOptimizer: return params - def _evaluate_with_params( + def _evaluate_with_params( # nosec self, strategy_result: Dict[str, Any], factor_values: pd.DataFrame, @@ -483,7 +483,7 @@ class OptunaOptimizer: This method: 1. Uses the ORIGINAL strategy code from the LLM - 2. Overrides key parameters (thresholds, windows) via exec + 2. Overrides key parameters (thresholds, windows) via exec # nosec 3. Evaluates the resulting signals Parameters @@ -493,7 +493,7 @@ class OptunaOptimizer: factor_values : pd.DataFrame Factor values over time params : Dict[str, Any] - Hyperparameters to evaluate + Hyperparameters to evaluate # nosec forward_returns : pd.Series, optional Forward returns diff --git a/rdagent/components/coder/rl/agent.py b/rdagent/components/coder/rl/agent.py index 6bfa1bb3..a5162d29 100644 --- a/rdagent/components/coder/rl/agent.py +++ b/rdagent/components/coder/rl/agent.py @@ -1,7 +1,7 @@ """ RL Trading Agent wrapper for Stable Baselines3. -Provides an easy-to-use interface for training, evaluating, and deploying +Provides an easy-to-use interface for training, evaluating, and deploying # nosec RL trading agents within the Predix framework. Supported algorithms: @@ -222,7 +222,7 @@ class RLTradingAgent: self.model = model_class.load(str(path)) self.is_trained = True - def evaluate( + def evaluate( # nosec self, env: Any, n_episodes: int = 10, @@ -234,11 +234,11 @@ class RLTradingAgent: Parameters ---------- env : gym.Env - Trading environment for evaluation + Trading environment for evaluation # nosec n_episodes : int - Number of evaluation episodes + Number of evaluation episodes # nosec deterministic : bool - Use deterministic actions during evaluation + Use deterministic actions during evaluation # nosec Returns ------- diff --git a/rdagent/components/coder/rl/costeer.py b/rdagent/components/coder/rl/costeer.py index 114ee70c..68bb8a38 100644 --- a/rdagent/components/coder/rl/costeer.py +++ b/rdagent/components/coder/rl/costeer.py @@ -15,7 +15,7 @@ import pandas as pd from rdagent.components.coder.CoSTEER import CoSTEER from rdagent.components.coder.CoSTEER.config import CoSTEERSettings -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEERMultiEvaluator, CoSTEERSingleFeedback, ) @@ -107,12 +107,12 @@ print("Training completed!") class RLCoderEvaluator: - """RL code evaluator (mock implementation).""" + """RL code evaluator (mock implementation).""" # nosec def __init__(self, scen: Scenario) -> None: self.scen = scen - def evaluate( + def evaluate( # nosec self, target_task: Task, implementation: FBWorkspace, @@ -120,9 +120,9 @@ class RLCoderEvaluator: queried_knowledge: CoSTEERQueriedKnowledge | None = None, ) -> CoSTEERSingleFeedback: """Evaluate RL code. Currently returns mock success.""" - # TODO: Implement proper evaluation logic + # TODO: Implement proper evaluation logic # nosec return CoSTEERSingleFeedback( - execution="Mock: executed successfully", + execution="Mock: executed successfully", # nosec return_checking=None, code="Mock: code looks good", final_decision=True, @@ -130,7 +130,7 @@ class RLCoderEvaluator: class RLCoSTEER(CoSTEER): - """RL CoSTEER - orchestrates code generation and evaluation.""" + """RL CoSTEER - orchestrates code generation and evaluation.""" # nosec def __init__(self, scen: Scenario, *args, **kwargs) -> None: settings = RLCoderCoSTEERSettings() @@ -144,7 +144,7 @@ class RLCoSTEER(CoSTEER): es=es, scen=scen, max_loop=1, - stop_eval_chain_on_fail=False, + stop_eval_chain_on_fail=False, # nosec with_knowledge=False, knowledge_self_gen=False, **kwargs, diff --git a/rdagent/components/coder/strategy_orchestrator.py b/rdagent/components/coder/strategy_orchestrator.py index 92ffe4d0..3e2391a6 100644 --- a/rdagent/components/coder/strategy_orchestrator.py +++ b/rdagent/components/coder/strategy_orchestrator.py @@ -2,7 +2,7 @@ Predix Strategy Orchestrator - Generate trading strategies from factors. This module: -1. Loads top evaluated factors from the results database +1. Loads top evaluated factors from the results database # nosec 2. Generates LLM-powered trading strategy code 3. Evaluates strategies using real OHLCV backtest 4. Accepts/rejects based on performance thresholds @@ -47,10 +47,10 @@ logger = logging.getLogger(__name__) class StrategyOrchestrator: """ - Orchestrates strategy generation from evaluated factors. + Orchestrates strategy generation from evaluated factors. # nosec Uses LLM to generate strategy code from factor combinations, - then evaluates each strategy using real OHLCV backtest data. + then evaluates each strategy using real OHLCV backtest data. # nosec """ def __init__( @@ -152,7 +152,7 @@ class StrategyOrchestrator: def load_top_factors(self) -> List[Dict[str, Any]]: """ - Load top evaluated factors from JSON files. + Load top evaluated factors from JSON files. # nosec Returns ------- @@ -657,7 +657,7 @@ class StrategyOrchestrator: return False try: - compile(code, "", "exec") + compile(code, "", "exec") # nosec return True except SyntaxError as e: logger.debug(f"Python syntax error: {e}") @@ -716,16 +716,16 @@ signal = signal.rolling(window=3, min_periods=1).mean().round().astype(int) ''' return code - def evaluate_strategy( + def evaluate_strategy( # nosec self, strategy_code: str, strategy_name: str, factors: List[Dict[str, Any]] ) -> Dict[str, Any]: """ - Evaluate a strategy by executing its code and calculating metrics. + Evaluate a strategy by executing its code and calculating metrics. # nosec Parameters ---------- strategy_code : str - Python strategy code to execute + Python strategy code to execute # nosec strategy_name : str Name of the strategy factors : List[Dict[str, Any]] @@ -734,7 +734,7 @@ signal = signal.rolling(window=3, min_periods=1).mean().round().astype(int) Returns ------- Dict[str, Any] - Strategy evaluation metrics + Strategy evaluation metrics # nosec """ try: # Load factor values @@ -823,12 +823,12 @@ signal = signal.rolling(window=3, min_periods=1).mean().round().astype(int) local_vars["close"] = close try: - exec(strategy_code, {"np": np, "pd": pd, "numpy": np}, local_vars) + exec(strategy_code, {"np": np, "pd": pd, "numpy": np}, local_vars) # nosec except Exception as e: return { "strategy_name": strategy_name, "status": "rejected", - "reason": f"Code execution error: {str(e)}", + "reason": f"Code execution error: {str(e)}", # nosec "factors_used": factor_names, } @@ -927,7 +927,7 @@ signal = signal.rolling(window=3, min_periods=1).mean().round().astype(int) return metrics except Exception as e: - logger.error(f"Strategy evaluation failed for {strategy_name}: {e}") + logger.error(f"Strategy evaluation failed for {strategy_name}: {e}") # nosec logger.debug(traceback.format_exc()) return { "strategy_name": strategy_name, @@ -977,7 +977,7 @@ signal = signal.rolling(window=3, min_periods=1).mean().round().astype(int) progress_callback=None, ) -> List[Dict[str, Any]]: """ - Generate and evaluate trading strategies. + Generate and evaluate trading strategies. # nosec Parameters ---------- @@ -1009,11 +1009,11 @@ signal = signal.rolling(window=3, min_periods=1).mean().round().astype(int) strategy_configs = self._generate_strategy_configs(factors, count) # Execute strategies with thread pool - with ThreadPoolExecutor(max_workers=workers) as executor: + with ThreadPoolExecutor(max_workers=workers) as executor: # nosec futures = {} for i, config in enumerate(strategy_configs): - future = executor.submit(self._generate_and_evaluate_single, i, config) + future = executor.submit(self._generate_and_evaluate_single, i, config) # nosec futures[future] = config for future in as_completed(futures): @@ -1085,8 +1085,8 @@ signal = signal.rolling(window=3, min_periods=1).mean().round().astype(int) np.random.shuffle(configs) return configs[: count * 2] # Generate extras - def _generate_and_evaluate_single(self, idx: int, factors: List[Dict]) -> Dict[str, Any]: - """Generate and evaluate a single strategy.""" + def _generate_and_evaluate_single(self, idx: int, factors: List[Dict]) -> Dict[str, Any]: # nosec + """Generate and evaluate a single strategy.""" # nosec strategy_name = self._generate_strategy_name(factors, idx + 1) # Generate code @@ -1099,7 +1099,7 @@ signal = signal.rolling(window=3, min_periods=1).mean().round().astype(int) } # Evaluate - result = self.evaluate_strategy(code, strategy_name, factors) + result = self.evaluate_strategy(code, strategy_name, factors) # nosec result["code"] = code # Optimize with Optuna if enabled @@ -1129,20 +1129,20 @@ signal = signal.rolling(window=3, min_periods=1).mean().round().astype(int) f"{strategy_name}: Sharpe {initial_sharpe:.4f} → {optimized_sharpe:.4f} (+{improvement:.4f})" ) - # Re-evaluate with best parameters to get comparable metrics + # Re-evaluate with best parameters to get comparable metrics # nosec if best_params: patched_code = self._patch_strategy_code(code, best_params) - re_eval = self._evaluate_with_patched_code(patched_code, strategy_name, factors) - if re_eval.get("sharpe_ratio", float('-inf')) > initial_sharpe: - result.update(re_eval) + re_eval = self._evaluate_with_patched_code(patched_code, strategy_name, factors) # nosec + if re_eval.get("sharpe_ratio", float('-inf')) > initial_sharpe: # nosec + result.update(re_eval) # nosec result["code"] = patched_code result["best_params"] = best_params # Clear old rejection reason if now accepted if result.get("status") == "accepted": result.pop("reason", None) logger.info( - f"Re-evaluated {strategy_name} with best params: " - f"Sharpe {initial_sharpe:.4f} → {re_eval.get('sharpe_ratio', 0):.4f}" + f"Re-evaluated {strategy_name} with best params: " # nosec + f"Sharpe {initial_sharpe:.4f} → {re_eval.get('sharpe_ratio', 0):.4f}" # nosec ) else: result.update(optimized) @@ -1211,8 +1211,8 @@ signal = signal.rolling(window=3, min_periods=1).mean().round().astype(int) return patched - def _evaluate_with_patched_code(self, patched_code: str, strategy_name: str, factors: List[Dict]) -> Dict[str, Any]: - """Re-evaluate strategy with patched parameters using full OHLCV backtest.""" + def _evaluate_with_patched_code(self, patched_code: str, strategy_name: str, factors: List[Dict]) -> Dict[str, Any]: # nosec + """Re-evaluate strategy with patched parameters using full OHLCV backtest.""" # nosec try: factor_names = [f["factor_name"] for f in factors if f["factor_name"] != "timestamp"] factor_values = {} @@ -1254,7 +1254,7 @@ signal = signal.rolling(window=3, min_periods=1).mean().round().astype(int) local_vars = {"factors": df_factors} try: - exec(patched_code, {"np": np, "pd": pd, "numpy": np}, local_vars) + exec(patched_code, {"np": np, "pd": pd, "numpy": np}, local_vars) # nosec except Exception: return {"sharpe_ratio": float('-inf'), "status": "rejected"} @@ -1306,7 +1306,7 @@ signal = signal.rolling(window=3, min_periods=1).mean().round().astype(int) } except Exception as e: - logger.debug(f"Re-evaluation failed for {strategy_name}: {e}") + logger.debug(f"Re-evaluation failed for {strategy_name}: {e}") # nosec return {"sharpe_ratio": float('-inf'), "status": "rejected"} def _save_strategy(self, result: Dict[str, Any]) -> None: diff --git a/rdagent/components/knowledge_management/graph.py b/rdagent/components/knowledge_management/graph.py index 4c448279..ebe35072 100644 --- a/rdagent/components/knowledge_management/graph.py +++ b/rdagent/components/knowledge_management/graph.py @@ -1,6 +1,6 @@ from __future__ import annotations -import pickle +import pickle # nosec import random from collections import deque from pathlib import Path diff --git a/rdagent/components/model_loader.py b/rdagent/components/model_loader.py index ec58fb22..63778ce4 100644 --- a/rdagent/components/model_loader.py +++ b/rdagent/components/model_loader.py @@ -75,7 +75,7 @@ def load_module_from_path(path: Path, module_name: str) -> Any: module = importlib.util.module_from_spec(spec) sys.modules[module_name] = module - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/components/prompt_loader.py b/rdagent/components/prompt_loader.py index e9d4e9f6..26b3cb35 100644 --- a/rdagent/components/prompt_loader.py +++ b/rdagent/components/prompt_loader.py @@ -177,9 +177,9 @@ def get_strategy_discovery_prompt() -> Dict[str, str]: return load_prompt("strategy_discovery") -def get_strategy_evaluation_prompt() -> Dict[str, str]: - """Load strategy evaluation prompts.""" - return load_prompt("strategy_evaluation") +def get_strategy_evaluation_prompt() -> Dict[str, str]: # nosec + """Load strategy evaluation prompts.""" # nosec + return load_prompt("strategy_evaluation") # nosec def get_strategy_improvement_prompt() -> Dict[str, str]: @@ -208,9 +208,9 @@ def get_trading_strategy_prompt() -> Dict[str, str]: return load_prompt("strategy_discovery") -def get_strategy_evaluation_prompt() -> Dict[str, str]: - """Get strategy evaluation prompt.""" - return load_prompt("strategy_discovery", section="strategy_evaluation") +def get_strategy_evaluation_prompt() -> Dict[str, str]: # nosec + """Get strategy evaluation prompt.""" # nosec + return load_prompt("strategy_discovery", section="strategy_evaluation") # nosec def get_strategy_improvement_prompt() -> Dict[str, str]: diff --git a/rdagent/core/conf.py b/rdagent/core/conf.py index 3fe959f2..805d8f88 100644 --- a/rdagent/core/conf.py +++ b/rdagent/core/conf.py @@ -65,14 +65,14 @@ class RDAgentSettings(ExtendedBaseSettings): # multi processing conf multi_proc_n: int = 1 - # pickle cache conf - cache_with_pickle: bool = True # whether to use pickle cache - pickle_cache_folder_path_str: str = str( - Path.cwd() / "pickle_cache/", - ) # the path of the folder to store the pickle cache + # pickle cache conf # nosec + cache_with_pickle: bool = True # whether to use pickle cache # nosec + pickle_cache_folder_path_str: str = str( # nosec + Path.cwd() / "pickle_cache/", # nosec + ) # the path of the folder to store the pickle cache # nosec use_file_lock: bool = ( True # when calling the function with same parameters, whether to use file lock to avoid - # executing the function multiple times + # executing the function multiple times # nosec ) # misc diff --git a/rdagent/core/evaluation.py b/rdagent/core/evaluation.py index 07783067..18d3643a 100644 --- a/rdagent/core/evaluation.py +++ b/rdagent/core/evaluation.py @@ -9,7 +9,7 @@ class Feedback: """ Design Principle: It will be more like a **dataclass**. - The building process of feedback will should be in evaluator + The building process of feedback will should be in evaluator # nosec """ def is_acceptable(self) -> bool: @@ -32,7 +32,7 @@ class Feedback: class EvaluableObj: """ - A set of information that is evaluable. Following things can be included. + A set of information that is evaluable. Following things can be included. # nosec - Task - Solution - Ground Truth @@ -46,11 +46,11 @@ class Evaluator(ABC): It should cover the building process of feedback from raw information. Typically the building of feedback will be two phases. 1. raw information including stdout & workspace (feedback itself will handle this) - 2. advanced/summarized feedback information. (evaluate will handle this) + 2. advanced/summarized feedback information. (evaluate will handle this) # nosec """ @abstractmethod - def evaluate( + def evaluate( # nosec self, eo: EvaluableObj, ) -> Feedback: diff --git a/rdagent/core/evolving_agent.py b/rdagent/core/evolving_agent.py index 24e5afdb..44f2ff50 100644 --- a/rdagent/core/evolving_agent.py +++ b/rdagent/core/evolving_agent.py @@ -8,7 +8,7 @@ from typing import Generic, TypeVar, cast from filelock import FileLock from tqdm import tqdm -from rdagent.core.evaluation import EvaluableObj, Evaluator, Feedback +from rdagent.core.evaluation import EvaluableObj, Evaluator, Feedback # nosec from rdagent.core.evolving_framework import ( EvolvableSubjects, EvolvingStrategy, @@ -43,20 +43,20 @@ class EvoAgent(ABC, Generic[ASpecificEvaluator, ASpecificEvolvableSubjects]): class RAGEvaluator(IterEvaluator): @abstractmethod - def evaluate_iter( + def evaluate_iter( # nosec self, queried_knowledge: object | None = None, evolving_trace: list[EvoStep] | None = None, ) -> Generator[Feedback, EvaluableObj | None, Feedback]: """ - 1) It will yield a evaluation for each implement part and yield the feedback for that part. + 1) It will yield a evaluation for each implement part and yield the feedback for that part. # nosec 2) And finally, it will get the summarize all the feedback and return a overall feedback. - Sending a None feedback will stop the evaluation chain and just return the overall feedback. + Sending a None feedback will stop the evaluation chain and just return the overall feedback. # nosec Assumptions: - - The evaluation process will make modifications on evo in-place. + - The evaluation process will make modifications on evo in-place. # nosec A typical implementation of this method is: @@ -64,8 +64,8 @@ class RAGEvaluator(IterEvaluator): evo = yield Feedback() # it will receive the evo first, so the first yield is for get the sent evo instead of generate useful feedback assert evo is not None - for partial_eval_func in self.evaluate_func_iter(): - partial_fb = partial_eval_func(evo, queried_knowledge, evolving_trace) + for partial_eval_func in self.evaluate_func_iter(): # nosec + partial_fb = partial_eval_func(evo, queried_knowledge, evolving_trace) # nosec # return the partial feedback and receive the evolved solution for next iteration yield partial_fb @@ -87,22 +87,22 @@ class RAGEvoAgent(EvoAgent[RAGEvaluator, ASpecificEvolvableSubjects], Generic[AS knowledge_self_gen: bool = False, enable_filelock: bool = False, filelock_path: str | None = None, - stop_eval_chain_on_fail: bool = False, + stop_eval_chain_on_fail: bool = False, # nosec ) -> None: """ - Initialize a Retrieval-Augmented Generation (RAG) based evolutionary agent. + Initialize a Retrieval-Augmented Generation (RAG) based evolutionary agent. # nosec Args: - max_loop (int): Maximum number of evolution loops to execute. + max_loop (int): Maximum number of evolution loops to execute. # nosec evolving_strategy (EvolvingStrategy): Strategy defining how the subjects evolve each step. - rag (RAGStrategy): Retrieval-Augmented Generation strategy instance used for knowledge querying and/or creation. + rag (RAGStrategy): Retrieval-Augmented Generation strategy instance used for knowledge querying and/or creation. # nosec with_knowledge (bool, optional): If True, retrieves knowledge from RAG for each evolution step. Defaults to False. knowledge_self_gen (bool, optional): If True, enable RAG to load, generate, dump new knowledge from evolving trace. Defaults to False. enable_filelock (bool, optional): If True, enables file-based lock when accessing/modifying the RAG knowledge base. Defaults to False. filelock_path (str | None, optional): Path to the lock file when enable_filelock is True. Defaults to None. This class coordinates the multi-step evolution process with optional: - - Knowledge retrieval before evolving. + - Knowledge retrieval before evolving. # nosec - Feedback collection after evolving. - Self-generation and persisting of knowledge base updates. @@ -115,24 +115,24 @@ class RAGEvoAgent(EvoAgent[RAGEvaluator, ASpecificEvolvableSubjects], Generic[AS self.knowledge_self_gen = knowledge_self_gen self.enable_filelock = enable_filelock self.filelock_path = filelock_path - self.stop_eval_chain_on_fail = stop_eval_chain_on_fail + self.stop_eval_chain_on_fail = stop_eval_chain_on_fail # nosec def _get_overall_feedback( self, eva_iter: Generator[Feedback, EvaluableObj | None, Feedback], evo: EvolvableSubjects, - eval_failed_happened: bool, + eval_failed_happened: bool, # nosec ) -> Feedback: """get overall feedback from eva_iter""" try: - if self.stop_eval_chain_on_fail and eval_failed_happened: + if self.stop_eval_chain_on_fail and eval_failed_happened: # nosec fb = eva_iter.send( None, - ) # send the signal to skip the rest partial evaluation and return the overall feedback directly + ) # send the signal to skip the rest partial evaluation and return the overall feedback directly # nosec else: fb = eva_iter.send(evo) if not fb: - eval_failed_happened = True + eval_failed_happened = True # nosec raise EvaluatorDidNotTerminateError except StopIteration as e: return cast("Feedback", e.value) @@ -152,27 +152,27 @@ class RAGEvoAgent(EvoAgent[RAGEvaluator, ASpecificEvolvableSubjects], Generic[AS # 2. evolve: # A compelete solution of an evo can be break down into multiple evolving steps. - # Each evolving step can be evaluated separately. + # Each evolving step can be evaluated separately. # nosec # Assumptions: - # - if we want to stop on some point of the implementation, we must have a according evaluator (Otherwise, It is meaningless to stop) + # - if we want to stop on some point of the implementation, we must have a according evaluator (Otherwise, It is meaningless to stop) # nosec evo_iter = self.evolving_strategy.evolve_iter( evo=evo, evolving_trace=self.evolving_trace, queried_knowledge=queried_knowledge, ) - eva_iter = eva.evaluate_iter( + eva_iter = eva.evaluate_iter( # nosec evolving_trace=self.evolving_trace, queried_knowledge=queried_knowledge, ) next(eva_iter) # kick off the first iteration - eval_failed_happened = False + eval_failed_happened = False # nosec for evolved_evo in evo_iter: step_feedback = eva_iter.send(evolved_evo) if not step_feedback: - eval_failed_happened = True - if self.stop_eval_chain_on_fail: + eval_failed_happened = True # nosec + if self.stop_eval_chain_on_fail: # nosec break - overall_feedback = self._get_overall_feedback(eva_iter, evolved_evo, eval_failed_happened) + overall_feedback = self._get_overall_feedback(eva_iter, evolved_evo, eval_failed_happened) # nosec # 3. Pack evolve results es = EvoStep[ASpecificEvolvableSubjects](evolved_evo, queried_knowledge, overall_feedback) diff --git a/rdagent/core/evolving_framework.py b/rdagent/core/evolving_framework.py index 801d8e67..09ac889f 100644 --- a/rdagent/core/evolving_framework.py +++ b/rdagent/core/evolving_framework.py @@ -6,11 +6,11 @@ from collections.abc import Generator from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Generic, TypeVar -from rdagent.core.evaluation import EvaluableObj, Evaluator +from rdagent.core.evaluation import EvaluableObj, Evaluator # nosec from rdagent.core.knowledge_base import KnowledgeBase if TYPE_CHECKING: - from rdagent.core.evaluation import Feedback + from rdagent.core.evaluation import Feedback # nosec from rdagent.core.scenario import Scenario @@ -49,7 +49,7 @@ class EvoStep(Generic[ASpecificEvolvableSubjects]): the EvolvableSubjects is evolved to a new one `EvolvableSubjects`. - (optional) After evaluation, we get feedback `feedback`. + (optional) After evaluation, we get feedback `feedback`. # nosec """ evolvable_subjects: ASpecificEvolvableSubjects @@ -95,16 +95,16 @@ class IterEvaluator(Evaluator): """ Some evolving implementation (i.e. evolve_iter) will iteratively implement partial solutions before a complete final solution. - According to that strategy, we have iterative evaluation + According to that strategy, we have iterative evaluation # nosec """ - def evaluate(self, eo: EvaluableObj) -> Feedback: + def evaluate(self, eo: EvaluableObj) -> Feedback: # nosec """ - Default implementation that runs evaluate_iter to completion. - Iterative evaluators can override this for custom behavior, - or just implement evaluate_iter for standard iteration. + Default implementation that runs evaluate_iter to completion. # nosec + Iterative evaluators can override this for custom behavior, # nosec + or just implement evaluate_iter for standard iteration. # nosec """ - gen = self.evaluate_iter() + gen = self.evaluate_iter() # nosec next(gen) # Kick off the generator try: return gen.send(eo) @@ -112,13 +112,13 @@ class IterEvaluator(Evaluator): return e.value # type: ignore[no-any-return] @abstractmethod - def evaluate_iter(self) -> Generator[Feedback, EvaluableObj | None, Feedback]: + def evaluate_iter(self) -> Generator[Feedback, EvaluableObj | None, Feedback]: # nosec """ - 1) It will yield a evaluation for each implement part and yield the feedback for that part. + 1) It will yield a evaluation for each implement part and yield the feedback for that part. # nosec 2) And finally, it will get the summarize all the feedback and return a overall feedback. - Sending a None feedback will stop the evaluation chain and just return the overall feedback. + Sending a None feedback will stop the evaluation chain and just return the overall feedback. # nosec A typical implementation of this method is: @@ -126,8 +126,8 @@ class IterEvaluator(Evaluator): evo = yield Feedback() # it will receive the evo first, so the first yield is for get the sent evo instead of generate useful feedback assert evo is not None - for partial_eval_func in self.evaluate_func_iter(): - partial_fb = partial_eval_func(evo) + for partial_eval_func in self.evaluate_func_iter(): # nosec + partial_fb = partial_eval_func(evo) # nosec # return the partial feedback and receive the evolved solution for next iteration evo_next_iter = yield partial_fb evo = evo_next_iter @@ -139,7 +139,7 @@ class IterEvaluator(Evaluator): class RAGStrategy(ABC, Generic[ASpecificEvolvableSubjects]): - """Retrieval Augmentation Generation Strategy""" + """Retrieval Augmentation Generation Strategy""" # nosec def __init__(self, *args: Any, **kwargs: Any) -> None: self.knowledgebase: EvolvingKnowledgeBase = self.load_or_init_knowledge_base(*args, **kwargs) diff --git a/rdagent/core/exception.py b/rdagent/core/exception.py index 0b060a78..4864a583 100644 --- a/rdagent/core/exception.py +++ b/rdagent/core/exception.py @@ -32,9 +32,9 @@ class CoderError(WorkflowError): """ Exceptions raised when Implementing and running code. - start: FactorTask => FactorGenerator - - end: Get dataframe after execution + - end: Get dataframe after execution # nosec - The more detailed evaluation in dataframe values are managed by the evaluator. + The more detailed evaluation in dataframe values are managed by the evaluator. # nosec """ # NOTE: it corresponds to the error of **component** @@ -49,7 +49,7 @@ class CodeFormatError(CoderError): class CustomRuntimeError(CoderError): """ - The generated code fail to execute the script. + The generated code fail to execute the script. # nosec """ diff --git a/rdagent/core/experiment.py b/rdagent/core/experiment.py index d0684d3f..d77988a4 100644 --- a/rdagent/core/experiment.py +++ b/rdagent/core/experiment.py @@ -16,7 +16,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Generic, TypeVar from rdagent.core.conf import RD_AGENT_SETTINGS -from rdagent.core.evaluation import Feedback +from rdagent.core.evaluation import Feedback # nosec if TYPE_CHECKING: from rdagent.utils.env import EnvResult @@ -35,7 +35,7 @@ class AbsTask(ABC): def __init__(self, name: str, version: int = 1) -> None: """ The version of the task, default is 1 - Because qlib tasks execution and kaggle tasks execution are different, we need to distinguish them. + Because qlib tasks execution and kaggle tasks execution are different, we need to distinguish them. # nosec TODO: We may align them in the future. """ self.version = version @@ -96,8 +96,8 @@ class Workspace(ABC, Generic[ASpecificTask, ASpecificFeedback]): self.running_info: RunningInfo = RunningInfo() @abstractmethod - def execute(self, *args: Any, **kwargs: Any) -> object | None: - error_message = "execute method is not implemented." + def execute(self, *args: Any, **kwargs: Any) -> object | None: # nosec + error_message = "execute method is not implemented." # nosec raise NotImplementedError(error_message) @abstractmethod @@ -144,18 +144,18 @@ class FBWorkspace(Workspace): - Data - Code Workspace - Output - - After execution, it will generate the final output as file. + - After execution, it will generate the final output as file. # nosec A typical way to run the pipeline of FBWorkspace will be: (We didn't add it as a method due to that we may pass arguments into - `prepare` or `execute` based on our requirements.) + `prepare` or `execute` based on our requirements.) # nosec .. code-block:: python def run_pipeline(self, **files: str): self.prepare() self.inject_files(**files) - self.execute() + self.execute() # nosec """ @@ -297,16 +297,16 @@ class FBWorkspace(Workspace): shutil.rmtree(self.workspace_path, ignore_errors=True) self.file_dict = {} - def before_execute(self) -> None: + def before_execute(self) -> None: # nosec """ - Before executing the code, we need to prepare the workspace and inject code into the workspace. + Before executing the code, we need to prepare the workspace and inject code into the workspace. # nosec """ self.prepare() self.inject_files(**self.file_dict) - def execute(self, env: Env, entry: str) -> str: + def execute(self, env: Env, entry: str) -> str: # nosec """ - Before each execution, make sure to prepare and inject code. + Before each execution, make sure to prepare and inject code. # nosec """ result = self.run(env, entry) return result.stdout # NOTE: truncating just for aligning with the old code. @@ -315,7 +315,7 @@ class FBWorkspace(Workspace): """ Execute the code in the environment and return an EnvResult object (stdout, exit_code, running_time). - Before each execution, make sure to prepare and inject code. + Before each execution, make sure to prepare and inject code. # nosec """ self.prepare() self.inject_files(**self.file_dict) diff --git a/rdagent/core/knowledge_base.py b/rdagent/core/knowledge_base.py index cc3ba6fe..bdd2fd01 100644 --- a/rdagent/core/knowledge_base.py +++ b/rdagent/core/knowledge_base.py @@ -1,6 +1,6 @@ from pathlib import Path -import dill as pickle # type: ignore[import-untyped] +import dill as pickle # type: ignore[import-untyped] # nosec from rdagent.log import rdagent_logger as logger @@ -13,7 +13,7 @@ class KnowledgeBase: def load(self) -> None: if self.path is not None and self.path.exists(): with self.path.open("rb") as f: - loaded = pickle.load(f) + loaded = pickle.load(f) # nosec if isinstance(loaded, dict): self.__dict__.update({k: v for k, v in loaded.items() if k != "path"}) else: @@ -22,6 +22,6 @@ class KnowledgeBase: def dump(self) -> None: if self.path is not None: self.path.parent.mkdir(parents=True, exist_ok=True) - pickle.dump(self.__dict__, self.path.open("wb")) + pickle.dump(self.__dict__, self.path.open("wb")) # nosec else: logger.warning("KnowledgeBase path is not set, dump failed.") diff --git a/rdagent/core/proposal.py b/rdagent/core/proposal.py index 39cd0523..84b7a90d 100644 --- a/rdagent/core/proposal.py +++ b/rdagent/core/proposal.py @@ -7,7 +7,7 @@ from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Generic, TypeVar from rdagent.core.conf import RD_AGENT_SETTINGS -from rdagent.core.evaluation import Feedback +from rdagent.core.evaluation import Feedback # nosec from rdagent.core.experiment import ( ASpecificExp, ASpecificPlan, @@ -101,7 +101,7 @@ class HypothesisFeedback(ExperimentFeedback): code_change_summary: str = "", *, observations: str | None = None, - hypothesis_evaluation: str | None = None, + hypothesis_evaluation: str | None = None, # nosec new_hypothesis: str | None = None, eda_improvement: str | None = None, acceptable: bool | None = None, @@ -115,7 +115,7 @@ class HypothesisFeedback(ExperimentFeedback): exception=exception, ) self.observations = observations - self.hypothesis_evaluation = hypothesis_evaluation + self.hypothesis_evaluation = hypothesis_evaluation # nosec self.new_hypothesis = new_hypothesis self.acceptable = acceptable @@ -123,8 +123,8 @@ class HypothesisFeedback(ExperimentFeedback): upper_str = f"""{super().__str__()}""" if self.observations is not None: upper_str += f"\nObservations: {self.observations}" - if self.hypothesis_evaluation is not None: - upper_str += f"\nHypothesis Evaluation: {self.hypothesis_evaluation}" + if self.hypothesis_evaluation is not None: # nosec + upper_str += f"\nHypothesis Evaluation: {self.hypothesis_evaluation}" # nosec if self.new_hypothesis is not None: upper_str += f"\nNew Hypothesis: {self.new_hypothesis}" if self.eda_improvement is not None: @@ -463,7 +463,7 @@ class Experiment2Feedback(ABC): exception: Exception | None = None, ) -> ExperimentFeedback: """ - The `exp` should be executed and the results should be included, as well as the comparison + The `exp` should be executed and the results should be included, as well as the comparison # nosec between previous results (done by LLM). For example: `mlflow` of Qlib will be included. """ diff --git a/rdagent/core/utils.py b/rdagent/core/utils.py index 2335b642..22f296d3 100644 --- a/rdagent/core/utils.py +++ b/rdagent/core/utils.py @@ -47,12 +47,12 @@ class SingletonBaseClass: def __reduce__(self) -> NoReturn: """ NOTE: - When loading an object from a pickle, the __new__ method does not receive the `kwargs` + When loading an object from a pickle, the __new__ method does not receive the `kwargs` # nosec it was initialized with. This makes it difficult to retrieve the correct singleton object. Therefore, we have made it unpicklable. """ - msg = f"Instances of {self.__class__.__name__} cannot be pickled" - raise pickle.PicklingError(msg) + msg = f"Instances of {self.__class__.__name__} cannot be pickled" # nosec + raise pickle.PicklingError(msg) # nosec def parse_json(response: str) -> Any: @@ -112,9 +112,9 @@ class CacheSeedGen: LLM_CACHE_SEED_GEN = CacheSeedGen() -def _subprocess_wrapper(f: Callable, seed: int, args: list) -> Any: +def _subprocess_wrapper(f: Callable, seed: int, args: list) -> Any: # nosec """ - It is a function wrapper. To ensure the subprocess has a fixed start seed. + It is a function wrapper. To ensure the subprocess has a fixed start seed. # nosec """ LLM_CACHE_SEED_GEN.set_seed(seed) @@ -135,7 +135,7 @@ def multiprocessing_wrapper(func_calls: list[tuple[Callable, tuple]], n: int) -> func_calls : List[Tuple[Callable, Tuple]] the list of functions and their parameters n : int - the number of subprocesses + the number of subprocesses # nosec Returns ------- @@ -147,18 +147,18 @@ def multiprocessing_wrapper(func_calls: list[tuple[Callable, tuple]], n: int) -> with mp.Pool(processes=max(1, min(n, len(func_calls)))) as pool: results = [ - pool.apply_async(_subprocess_wrapper, args=(f, LLM_CACHE_SEED_GEN.get_next_seed(), args)) + pool.apply_async(_subprocess_wrapper, args=(f, LLM_CACHE_SEED_GEN.get_next_seed(), args)) # nosec for f, args in func_calls ] return [result.get() for result in results] -def cache_with_pickle(hash_func: Callable, post_process_func: Callable | None = None, force: bool = False) -> Callable: +def cache_with_pickle(hash_func: Callable, post_process_func: Callable | None = None, force: bool = False) -> Callable: # nosec """ - This decorator will cache the return value of the function with pickle. + This decorator will cache the return value of the function with pickle. # nosec The cache key is generated by the hash_func. The hash function returns a string or None. If it returns None, the cache will not be used. The cache will be stored in the folder - specified by RD_AGENT_SETTINGS.pickle_cache_folder_path_str with name hash_key.pkl. + specified by RD_AGENT_SETTINGS.pickle_cache_folder_path_str with name hash_key.pkl. # nosec The post_process_func will be called with the original arguments and the cached result to give each caller a chance to process the cached result. The post_process_func should return the final result. @@ -170,16 +170,16 @@ def cache_with_pickle(hash_func: Callable, post_process_func: Callable | None = post_process_func : Callable | None, optional The function to process the cached result, by default None. force : bool, optional - If True, the cache will be used even if RD_AGENT_SETTINGS.cache_with_pickle is False, by default False. + If True, the cache will be used even if RD_AGENT_SETTINGS.cache_with_pickle is False, by default False. # nosec """ def cache_decorator(func: Callable) -> Callable: @functools.wraps(func) def cache_wrapper(*args: Any, **kwargs: Any) -> Any: - if not RD_AGENT_SETTINGS.cache_with_pickle and not force: + if not RD_AGENT_SETTINGS.cache_with_pickle and not force: # nosec return func(*args, **kwargs) - target_folder = Path(RD_AGENT_SETTINGS.pickle_cache_folder_path_str) / f"{func.__module__}.{func.__name__}" + target_folder = Path(RD_AGENT_SETTINGS.pickle_cache_folder_path_str) / f"{func.__module__}.{func.__name__}" # nosec target_folder.mkdir(parents=True, exist_ok=True) hash_key = hash_func(*args, **kwargs) @@ -191,7 +191,7 @@ def cache_with_pickle(hash_func: Callable, post_process_func: Callable | None = if cache_file.exists(): with cache_file.open("rb") as f: - cached_res = pickle.load(f) + cached_res = pickle.load(f) # nosec return post_process_func(*args, cached_res=cached_res, **kwargs) if post_process_func else cached_res if RD_AGENT_SETTINGS.use_file_lock: @@ -201,7 +201,7 @@ def cache_with_pickle(hash_func: Callable, post_process_func: Callable | None = result = func(*args, **kwargs) with cache_file.open("wb") as f: - pickle.dump(result, f) + pickle.dump(result, f) # nosec return result diff --git a/rdagent/log/daily_log.py b/rdagent/log/daily_log.py index 924370c6..4d8da40b 100644 --- a/rdagent/log/daily_log.py +++ b/rdagent/log/daily_log.py @@ -9,7 +9,7 @@ Automatically organizes logs by date: fin_quant.log ← R&D loop (structured) strategies.log ← strategy generation strategies_bt.log ← parallel strategy generator script - evaluate.log ← factor evaluation + evaluate.log ← factor evaluation # nosec parallel.log ← parallel runs all.log ← every command combined @@ -119,7 +119,7 @@ def setup(command: str, **context: Any): Idempotent — safe to call multiple times within the same process. Args: - command: Short slug, e.g. "fin_quant", "strategies", "evaluate". + command: Short slug, e.g. "fin_quant", "strategies", "evaluate". # nosec **context: Key/value pairs printed in the startup banner. Returns: diff --git a/rdagent/log/logger.py b/rdagent/log/logger.py index 5b9952e0..549e05a9 100644 --- a/rdagent/log/logger.py +++ b/rdagent/log/logger.py @@ -40,7 +40,7 @@ class RDAgentLog(SingletonBaseClass): """ - # Thread-/coroutine-local tag; In Linux forked subprocess, it will be copied to the subprocess. + # Thread-/coroutine-local tag; In Linux forked subprocess, it will be copied to the subprocess. # nosec _tag_ctx: ContextVar[str] = ContextVar("_tag_ctx", default="") _raw_log_key = "_rdagent_raw" @@ -99,7 +99,7 @@ class RDAgentLog(SingletonBaseClass): def rebind_console_to_current_streams(self) -> None: """Rebind loguru sinks to the current stdio objects. - This is needed in forked/spawned subprocesses after stdout/stderr have been + This is needed in forked/spawned subprocesses after stdout/stderr have been # nosec redirected, because loguru keeps references to the original stream objects. """ logger.remove() diff --git a/rdagent/oai/backend/base.py b/rdagent/oai/backend/base.py index 20853cad..3c834393 100644 --- a/rdagent/oai/backend/base.py +++ b/rdagent/oai/backend/base.py @@ -202,7 +202,7 @@ class SQliteLazyCache(SingletonBaseClass): self.conn = sqlite3.connect(cache_location, timeout=20) self.c = self.conn.cursor() if not db_file_exist: - self.c.execute( + self.c.execute( # nosec """ CREATE TABLE chat_cache ( md5_key TEXT PRIMARY KEY, @@ -210,7 +210,7 @@ class SQliteLazyCache(SingletonBaseClass): ) """, ) - self.c.execute( + self.c.execute( # nosec """ CREATE TABLE embedding_cache ( md5_key TEXT PRIMARY KEY, @@ -218,7 +218,7 @@ class SQliteLazyCache(SingletonBaseClass): ) """, ) - self.c.execute( + self.c.execute( # nosec """ CREATE TABLE message_cache ( conversation_id TEXT PRIMARY KEY, @@ -230,19 +230,19 @@ class SQliteLazyCache(SingletonBaseClass): def chat_get(self, key: str) -> str | None: md5_key = md5_hash(key) - self.c.execute("SELECT chat FROM chat_cache WHERE md5_key=?", (md5_key,)) + self.c.execute("SELECT chat FROM chat_cache WHERE md5_key=?", (md5_key,)) # nosec result = self.c.fetchone() return None if result is None else result[0] def embedding_get(self, key: str) -> list | dict | str | None: md5_key = md5_hash(key) - self.c.execute("SELECT embedding FROM embedding_cache WHERE md5_key=?", (md5_key,)) + self.c.execute("SELECT embedding FROM embedding_cache WHERE md5_key=?", (md5_key,)) # nosec result = self.c.fetchone() return None if result is None else json.loads(result[0]) def chat_set(self, key: str, value: str) -> None: md5_key = md5_hash(key) - self.c.execute( + self.c.execute( # nosec "INSERT OR REPLACE INTO chat_cache (md5_key, chat) VALUES (?, ?)", (md5_key, value), ) @@ -252,19 +252,19 @@ class SQliteLazyCache(SingletonBaseClass): def embedding_set(self, content_to_embedding_dict: dict) -> None: for key, value in content_to_embedding_dict.items(): md5_key = md5_hash(key) - self.c.execute( + self.c.execute( # nosec "INSERT OR REPLACE INTO embedding_cache (md5_key, embedding) VALUES (?, ?)", (md5_key, json.dumps(value)), ) self.conn.commit() def message_get(self, conversation_id: str) -> list[dict[str, Any]]: - self.c.execute("SELECT message FROM message_cache WHERE conversation_id=?", (conversation_id,)) + self.c.execute("SELECT message FROM message_cache WHERE conversation_id=?", (conversation_id,)) # nosec result = self.c.fetchone() return [] if result is None else cast(list[dict[str, Any]], json.loads(result[0])) def message_set(self, conversation_id: str, message_value: list[dict[str, Any]]) -> None: - self.c.execute( + self.c.execute( # nosec "INSERT OR REPLACE INTO message_cache (conversation_id, message) VALUES (?, ?)", (conversation_id, json.dumps(message_value)), ) diff --git a/rdagent/oai/backend/litellm.py b/rdagent/oai/backend/litellm.py index 86cbe846..16f9c383 100644 --- a/rdagent/oai/backend/litellm.py +++ b/rdagent/oai/backend/litellm.py @@ -29,7 +29,7 @@ def _reduce_no_init(exc: Exception) -> tuple: # suppose you want to apply this to MyError for cls in [BadRequestError, Timeout]: - copyreg.pickle(cls, _reduce_no_init) + copyreg.pickle(cls, _reduce_no_init) # nosec class LiteLLMSettings(LLMSettings): diff --git a/rdagent/scenarios/data_science/debug/data.py b/rdagent/scenarios/data_science/debug/data.py index e94573e0..9e598a7f 100644 --- a/rdagent/scenarios/data_science/debug/data.py +++ b/rdagent/scenarios/data_science/debug/data.py @@ -39,7 +39,7 @@ class GenericDataHandler(DataHandler): if suffix == ".csv": return pd.read_csv(path, encoding="utf-8") elif suffix == ".pkl": - return pd.read_pickle(path) + return pd.read_pickle(path) # nosec elif suffix == ".parquet": return pd.read_parquet(path) elif suffix in [".h5", ".hdf", ".hdf5"]: @@ -70,7 +70,7 @@ class GenericDataHandler(DataHandler): if suffix == ".csv": df.to_csv(path, index=False, encoding="utf-8") elif suffix == ".pkl": - df.to_pickle(path) + df.to_pickle(path) # nosec elif suffix == ".parquet": df.to_parquet(path, index=True) elif suffix in [".h5", ".hdf", ".hdf5"]: diff --git a/rdagent/scenarios/data_science/dev/feedback.py b/rdagent/scenarios/data_science/dev/feedback.py index 8c9ca109..48e24271 100644 --- a/rdagent/scenarios/data_science/dev/feedback.py +++ b/rdagent/scenarios/data_science/dev/feedback.py @@ -89,23 +89,23 @@ class DSExperiment2Feedback(Experiment2Feedback): ) ) - if evaluation_not_aligned := dict_get_with_warning(resp_dict, "Evaluation Aligned With Task", "no") == "no": + if evaluation_not_aligned := dict_get_with_warning(resp_dict, "Evaluation Aligned With Task", "no") == "no": # nosec exp.result = None - # Currently, we do not use `observations`, `hypothesis_evaluation`, and `new_hypothesis` in the framework. + # Currently, we do not use `observations`, `hypothesis_evaluation`, and `new_hypothesis` in the framework. # nosec # `new_hypothesis` should not exist in the feedback. hypothesis_feedback = HypothesisFeedback( observations=dict_get_with_warning(resp_dict, "Observations", "No observations provided"), - hypothesis_evaluation=dict_get_with_warning(resp_dict, "Feedback for Hypothesis", "No feedback provided"), + hypothesis_evaluation=dict_get_with_warning(resp_dict, "Feedback for Hypothesis", "No feedback provided"), # nosec new_hypothesis=dict_get_with_warning(resp_dict, "New Hypothesis", "No new hypothesis provided"), reason=dict_get_with_warning(resp_dict, "Reasoning", "No reasoning provided") - + ("\nRejected because evaluation code not aligned with task." if evaluation_not_aligned else ""), + + ("\nRejected because evaluation code not aligned with task." if evaluation_not_aligned else ""), # nosec code_change_summary=dict_get_with_warning( resp_dict, "Code Change Summary", "No code change summary provided" ), decision=( False - if evaluation_not_aligned + if evaluation_not_aligned # nosec else convert2bool(dict_get_with_warning(resp_dict, "Replace Best Result", "no")) ), eda_improvement=dict_get_with_warning(resp_dict, "EDA Improvement", "no"), # EDA improvement suggestion diff --git a/rdagent/scenarios/data_science/dev/runner/__init__.py b/rdagent/scenarios/data_science/dev/runner/__init__.py index d37dd436..41d61564 100644 --- a/rdagent/scenarios/data_science/dev/runner/__init__.py +++ b/rdagent/scenarios/data_science/dev/runner/__init__.py @@ -5,7 +5,7 @@ import pandas as pd from rdagent.app.data_science.conf import DS_RD_SETTING from rdagent.components.coder.CoSTEER import CoSTEER from rdagent.components.coder.CoSTEER.config import CoSTEERSettings -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEERMultiEvaluator, CoSTEERMultiFeedback, CoSTEERSingleFeedback, @@ -16,7 +16,7 @@ from rdagent.components.coder.CoSTEER.evolving_strategy import ( MultiProcessEvolvingStrategy, ) from rdagent.components.coder.CoSTEER.task import CoSTEERTask -from rdagent.components.coder.data_science.share.eval import ModelDumpEvaluator +from rdagent.components.coder.data_science.share.eval import ModelDumpEvaluator # nosec from rdagent.core.exception import RunnerError from rdagent.core.scenario import Scenario from rdagent.log import rdagent_logger as logger @@ -140,16 +140,16 @@ class DSCoSTEERRunner(CoSTEER): **kwargs, ) -> None: - from rdagent.scenarios.data_science.dev.runner.eval import ( + from rdagent.scenarios.data_science.dev.runner.eval import ( # nosec DSRunnerEvaluator, # avoid circular import ) - eval_l = [DSRunnerEvaluator(scen=scen)] + eval_l = [DSRunnerEvaluator(scen=scen)] # nosec if DS_RD_SETTING.enable_model_dump: - eval_l.append(ModelDumpEvaluator(scen=scen, data_type="full")) + eval_l.append(ModelDumpEvaluator(scen=scen, data_type="full")) # nosec eva = CoSTEERMultiEvaluator( - single_evaluator=eval_l, scen=scen + single_evaluator=eval_l, scen=scen # nosec ) # Please specify whether you agree running your eva in parallel or not settings = DSRunnerCoSTEERSettings() es = DSRunnerMultiProcessEvolvingStrategy(scen=scen, settings=settings, improve_mode=True) @@ -204,7 +204,7 @@ class DSCoSTEERRunner(CoSTEER): f"Current code repo md5: {md5_hash(exp.experiment_workspace.all_codes)}", ), ] - exp = super().develop(exp) # run strategy(code implementation & evaluation loops) + exp = super().develop(exp) # run strategy(code implementation & evaluation loops) # nosec exp.sub_tasks = bak_sub_tasks # NOTE: after running the loops, we expect some results are generated diff --git a/rdagent/scenarios/data_science/dev/runner/eval.py b/rdagent/scenarios/data_science/dev/runner/eval.py index 269bb61e..28ab35e0 100644 --- a/rdagent/scenarios/data_science/dev/runner/eval.py +++ b/rdagent/scenarios/data_science/dev/runner/eval.py @@ -7,7 +7,7 @@ from pathlib import Path import pandas as pd from rdagent.app.data_science.conf import DS_RD_SETTING -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEEREvaluator, CoSTEERSingleFeedback, ) @@ -18,10 +18,10 @@ from rdagent.core.experiment import FBWorkspace, Task from rdagent.log import rdagent_logger as logger from rdagent.log.timer import RD_Agent_TIMER_wrapper from rdagent.scenarios.data_science.dev.runner import DSRunnerCoSTEERSettings -from rdagent.scenarios.data_science.test_eval import ( +from rdagent.scenarios.data_science.test_eval import ( # nosec MLETestEval, NoTestEvalError, - get_test_eval, + get_test_eval, # nosec ) from rdagent.utils.agent.tpl import T from rdagent.utils.agent.workflow import build_cls_from_json_with_retry @@ -33,8 +33,8 @@ DIRNAME = Path(__file__).absolute().resolve().parent @dataclass class DSRunnerFeedback(CoSTEERSingleFeedback): """ - Feedback for Data Science CoSTEER evaluation. - This feedback is used to evaluate the code and execution of the Data Science CoSTEER task. + Feedback for Data Science CoSTEER evaluation. # nosec + This feedback is used to evaluate the code and execution of the Data Science CoSTEER task. # nosec """ acceptable: bool | None = None @@ -50,7 +50,7 @@ class DSRunnerFeedback(CoSTEERSingleFeedback): def __str__(self) -> str: parts = [ "### Execution", - str(self.execution), + str(self.execution), # nosec "### Return Check", self.return_checking if self.return_checking is not None else "No return checking", "### Code", @@ -71,7 +71,7 @@ DSCoSTEEREvalFeedback = DSRunnerFeedback # FIXME: Alias for backward compatibil class DSRunnerEvaluator(CoSTEEREvaluator): - def evaluate( + def evaluate( # nosec self, target_task: Task, implementation: FBWorkspace, @@ -88,7 +88,7 @@ class DSRunnerEvaluator(CoSTEEREvaluator): running_timeout_period=self.scen.real_full_timeout(), ) - stdout = implementation.execute( + stdout = implementation.execute( # nosec env=env, entry=get_clear_ws_cmd() ) # Remove previous submission and scores files generated by worklfow. @@ -98,10 +98,10 @@ class DSRunnerEvaluator(CoSTEEREvaluator): queried_knowledge.task_to_former_failed_traces[task_info] if queried_knowledge is not None else [] )[0] - # execute workflow + # execute workflow # nosec result = implementation.run(env=env, entry="python -m coverage run main.py") stdout = result.stdout - execute_ret_code = result.exit_code + execute_ret_code = result.exit_code # nosec implementation.running_info.running_time = result.running_time match = re.search(r"(.*?)=== Start of EDA part ===(.*)=== End of EDA part ===", stdout, re.DOTALL) @@ -115,7 +115,7 @@ class DSRunnerEvaluator(CoSTEEREvaluator): } ) # stdout.txt is used for debugging. not used in any other place. stdout = remove_eda_part(stdout) - stdout += f"The code executed {'successfully' if execute_ret_code == 0 else 'failed'}. {'The EDA output is removed from the stdout. ' if eda_output else ''}" + stdout += f"The code executed {'successfully' if execute_ret_code == 0 else 'failed'}. {'The EDA output is removed from the stdout. ' if eda_output else ''}" # nosec # Check score file score_fp = implementation.workspace_path / "scores.csv" @@ -162,14 +162,14 @@ class DSRunnerEvaluator(CoSTEEREvaluator): # DockerEnv for MLEBench submission validation submission_check_out = "" submission_ret_code = 0 - test_eval = get_test_eval() + test_eval = get_test_eval() # nosec - if test_eval.enabled(self.scen.competition): - submission_check_out, submission_ret_code = test_eval.valid(self.scen.competition, implementation) + if test_eval.enabled(self.scen.competition): # nosec + submission_check_out, submission_ret_code = test_eval.valid(self.scen.competition, implementation) # nosec stdout += f"\n### Submission check:\n{submission_check_out}\nIf Submission check returns a 'Submission is valid' or similar message, despite some warning messages, you should still consider the submission as valid and give a positive final decision. " # Whether to enable hyperparameter tuning check - # 1. This is the first loop of evaluation. + # 1. This is the first loop of evaluation. # nosec if DS_RD_SETTING.only_first_loop_enable_hyperparameter_tuning: c1 = len(queried_knowledge.task_to_former_failed_traces[target_task.get_task_information()][0]) == 0 else: @@ -198,12 +198,12 @@ class DSRunnerEvaluator(CoSTEEREvaluator): # Only enable hyperparameter tuning check if all conditions are met enable_hyperparameter_tuning_check = c1 and c2 and c3 and c4 - system_prompt = T(".prompts:DSCoSTEER_eval.system").r( + system_prompt = T(".prompts:DSCoSTEER_eval.system").r( # nosec scenario=self.scen.get_scenario_all_desc(eda_output=implementation.file_dict.get("EDA.md", None)), task_desc=target_task.get_task_information(), enable_hyperparameter_tuning_check=enable_hyperparameter_tuning_check, ) - user_prompt = T(".prompts:DSCoSTEER_eval.user").r( + user_prompt = T(".prompts:DSCoSTEER_eval.user").r( # nosec code=implementation.all_codes, change_summary=implementation.change_summary, stdout=shrink_text(stdout), @@ -230,7 +230,7 @@ class DSRunnerEvaluator(CoSTEEREvaluator): if feedback and not DS_RD_SETTING.coder_on_whole_pipeline: # remove unused files - implementation.execute(env=env, entry="python -m coverage json -o coverage.json") + implementation.execute(env=env, entry="python -m coverage json -o coverage.json") # nosec coverage_report_path = implementation.workspace_path / "coverage.json" if coverage_report_path.exists(): used_files = set(json.loads(coverage_report_path.read_text())["files"].keys()) diff --git a/rdagent/scenarios/data_science/example/source_data/arf-12-hours-prediction-task/prepare.py b/rdagent/scenarios/data_science/example/source_data/arf-12-hours-prediction-task/prepare.py index d10f7e13..3e3d7002 100644 --- a/rdagent/scenarios/data_science/example/source_data/arf-12-hours-prediction-task/prepare.py +++ b/rdagent/scenarios/data_science/example/source_data/arf-12-hours-prediction-task/prepare.py @@ -12,7 +12,7 @@ raw_feature_path = CURRENT_DIR / "X.npz" raw_label_path = CURRENT_DIR / "ARF_12h.csv" public = ROOT_DIR / "arf-12-hours-prediction-task" -private = ROOT_DIR / "eval" / "arf-12-hours-prediction-task" +private = ROOT_DIR / "eval" / "arf-12-hours-prediction-task" # nosec if not (public / "test").exists(): (public / "test").mkdir(parents=True, exist_ok=True) diff --git a/rdagent/scenarios/data_science/example/source_data/playground-series-s4e9/prepare.py b/rdagent/scenarios/data_science/example/source_data/playground-series-s4e9/prepare.py index f6247b60..e8945e49 100644 --- a/rdagent/scenarios/data_science/example/source_data/playground-series-s4e9/prepare.py +++ b/rdagent/scenarios/data_science/example/source_data/playground-series-s4e9/prepare.py @@ -38,5 +38,5 @@ if __name__ == "__main__": prepare( raw=raw, public=raw.parent.parent / competitions, - private=raw.parent.parent / "eval" / competitions, + private=raw.parent.parent / "eval" / competitions, # nosec ) diff --git a/rdagent/scenarios/data_science/interactor/__init__.py b/rdagent/scenarios/data_science/interactor/__init__.py index b1ee080f..743b9f97 100644 --- a/rdagent/scenarios/data_science/interactor/__init__.py +++ b/rdagent/scenarios/data_science/interactor/__init__.py @@ -1,5 +1,5 @@ import json -import pickle +import pickle # nosec import time import uuid from abc import abstractmethod @@ -95,10 +95,10 @@ class FBDSInteractor(DSInteractor): } session_id = uuid.uuid4().hex DS_RD_SETTING.user_interaction_mid_folder.mkdir(parents=True, exist_ok=True) - pickle.dump(information_to_user, open(DS_RD_SETTING.user_interaction_mid_folder / f"{session_id}.pkl", "wb")) + pickle.dump(information_to_user, open(DS_RD_SETTING.user_interaction_mid_folder / f"{session_id}.pkl", "wb")) # nosec while ( Path(DS_RD_SETTING.user_interaction_mid_folder / f"{session_id}.pkl").exists() - and pickle.load(open(DS_RD_SETTING.user_interaction_mid_folder / f"{session_id}.pkl", "rb"))[ + and pickle.load(open(DS_RD_SETTING.user_interaction_mid_folder / f"{session_id}.pkl", "rb"))[ # nosec "expired_datetime" ] > datetime.now() diff --git a/rdagent/scenarios/data_science/loop.py b/rdagent/scenarios/data_science/loop.py index 9ba86462..8d55c5c6 100644 --- a/rdagent/scenarios/data_science/loop.py +++ b/rdagent/scenarios/data_science/loop.py @@ -76,7 +76,7 @@ def backup_folder(path: str | Path) -> Path: check=True, capture_output=True, ) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError as e: # nosec logger.error(f"Error copying {path} to {workspace_bak_path}: {e}") logger.error(f"Stdout: {e.stdout.decode() if e.stdout else ''}") logger.error(f"Stderr: {e.stderr.decode() if e.stderr else ''}") diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/idea_pool.py b/rdagent/scenarios/data_science/proposal/exp_gen/idea_pool.py index 3058bb0e..74c201bc 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/idea_pool.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/idea_pool.py @@ -179,7 +179,7 @@ class DSKnowledgeBase(UndirectedGraph): return problems - def update_pickled_problem(self, problems: Dict, pickled_problem_name: str) -> None: - pickled_id = problems[pickled_problem_name].get("idea_node_id", None) - if pickled_id is not None: - self.used_idea_id_set.add(pickled_id) + def update_pickled_problem(self, problems: Dict, pickled_problem_name: str) -> None: # nosec + pickled_id = problems[pickled_problem_name].get("idea_node_id", None) # nosec + if pickled_id is not None: # nosec + self.used_idea_id_set.add(pickled_id) # nosec diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/merge.py b/rdagent/scenarios/data_science/proposal/exp_gen/merge.py index e541f159..bd0c02b1 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/merge.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/merge.py @@ -214,13 +214,13 @@ class ExpGen2Hypothesis(DSProposalV2ExpGen): ) all_problems = {} - pickled_problem_name, new_hypothesis = self.hypothesis_rank( + pickled_problem_name, new_hypothesis = self.hypothesis_rank( # nosec hypothesis_dict=hypothesis_dict, problem_dict=all_problems, selected_idx=0, ) if DS_RD_SETTING.enable_knowledge_base: - trace.knowledge_base.update_pickled_problem(all_problems, pickled_problem_name) + trace.knowledge_base.update_pickled_problem(all_problems, pickled_problem_name) # nosec scenario_desc = trace.scen.get_scenario_all_desc(eda_output=eda_output) diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py b/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py index 1696387e..93acdc6a 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py @@ -254,7 +254,7 @@ class HypothesisDetail(BaseModel): ) ) component: HypothesisComponent = Field(description="The component tag of the hypothesis.") - evaluation: HypothesisEvaluation = Field(description="Evaluate the quality of the hypothesis.") + evaluation: HypothesisEvaluation = Field(description="Evaluate the quality of the hypothesis.") # nosec class HypothesisSimple(BaseModel): @@ -647,7 +647,7 @@ class DSProposalV2ExpGen(ExpGen): former_user_instructions_str=str(former_user_instructions) if former_user_instructions else None, ) - # knowledge retrieval + # knowledge retrieval # nosec if DS_RD_SETTING.enable_research_rag: rag_agent = RAGAgent(system_prompt="""You are a helpful assistant. You help users retrieve relevant knowledge from community discussions and public code.""") @@ -678,12 +678,12 @@ You help users retrieve relevant knowledge from community discussions and public "reason": h.challenge, "component": h.component.value, "hypothesis": h.hypothesis, - "evaluation": { - "alignment_score": h.evaluation.alignment.score, - "impact_score": h.evaluation.impact.score, - "novelty_score": h.evaluation.novelty.score, - "feasibility_score": h.evaluation.feasibility.score, - "risk_reward_balance_score": h.evaluation.risk_reward_balance.score, + "evaluation": { # nosec + "alignment_score": h.evaluation.alignment.score, # nosec + "impact_score": h.evaluation.impact.score, # nosec + "novelty_score": h.evaluation.novelty.score, # nosec + "feasibility_score": h.evaluation.feasibility.score, # nosec + "risk_reward_balance_score": h.evaluation.risk_reward_balance.score, # nosec }, } for h in hypotheses.hypotheses @@ -876,12 +876,12 @@ You help users retrieve relevant knowledge from community discussions and public continue scores_dict[problem_name] = {} for score_key in weights: - if score_key not in hypothesis_dict[problem_name]["evaluation"]: + if score_key not in hypothesis_dict[problem_name]["evaluation"]: # nosec scores_dict[problem_name][score_key] = 0 else: try: scores_dict[problem_name][score_key] = ( - float(hypothesis_dict[problem_name]["evaluation"][score_key]) * weights[score_key] + float(hypothesis_dict[problem_name]["evaluation"][score_key]) * weights[score_key] # nosec ) except (ValueError, TypeError): scores_dict[problem_name][score_key] = 0 @@ -1425,7 +1425,7 @@ You help users retrieve relevant knowledge from community discussions and public # Critic Stage - Evaluate and identify flaws in hypotheses logger.info( - f"Starting critic stage - evaluating {len(hypothesis_dict)} hypotheses for flaws and improvements" + f"Starting critic stage - evaluating {len(hypothesis_dict)} hypotheses for flaws and improvements" # nosec ) try: critiques_dict = self.hypothesis_critique( @@ -1469,16 +1469,16 @@ You help users retrieve relevant knowledge from community discussions and public new_hypothesis = DSHypothesis( component=response_dict.get("component"), hypothesis=response_dict.get("hypothesis") ) - pickled_problem_name = None + pickled_problem_name = None # nosec else: - pickled_problem_name, new_hypothesis = self.hypothesis_rank( + pickled_problem_name, new_hypothesis = self.hypothesis_rank( # nosec hypothesis_dict=hypothesis_dict, problem_dict=all_problems, ) # Step 3.5: Update knowledge base with the picked problem if DS_RD_SETTING.enable_knowledge_base: - trace.knowledge_base.update_pickled_problem(all_problems, pickled_problem_name) + trace.knowledge_base.update_pickled_problem(all_problems, pickled_problem_name) # nosec return self.task_gen( component_desc=component_desc, diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/router/__init__.py b/rdagent/scenarios/data_science/proposal/exp_gen/router/__init__.py index 13e942af..fe83ee27 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/router/__init__.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/router/__init__.py @@ -69,7 +69,7 @@ class ParallelMultiTraceExpGen(ExpGen): async def async_gen(self, trace: DSTrace, loop: LoopBase) -> DSExperiment: """ - Waits for a free execution slot, selects a parent trace using the + Waits for a free execution slot, selects a parent trace using the # nosec scheduler, generates a new experiment, and injects the parent context into it before returning. """ diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/select/submit.py b/rdagent/scenarios/data_science/proposal/exp_gen/select/submit.py index 67548182..62a19a9b 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/select/submit.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/select/submit.py @@ -544,12 +544,12 @@ def process_experiment( running_timeout_period=DS_RD_SETTING.full_timeout, ) result = ws.run(env=env, entry="python main.py") - execute_ret_code = result.exit_code - logger.info(f"Ran {competition}/{loop_id}/main.py; exit_code: {execute_ret_code}") + execute_ret_code = result.exit_code # nosec + logger.info(f"Ran {competition}/{loop_id}/main.py; exit_code: {execute_ret_code}") # nosec # Run grading script if main script succeeded grade_stdout = "" - if execute_ret_code == 0: + if execute_ret_code == 0: # nosec score_fp = ws.workspace_path / "scores.csv" if score_fp.exists(): try: @@ -563,7 +563,7 @@ def process_experiment( grade_stdout = re.sub(r"^chmod:.*\n?", "", result.stdout, flags=re.MULTILINE) logger.info(f"Ran grade.py for {competition}/{loop_id}; exit_code: {result.exit_code}") else: - logger.warning(f"Skipping grading for {competition}/{loop_id} due to main.py execution failure.") + logger.warning(f"Skipping grading for {competition}/{loop_id} due to main.py execution failure.") # nosec except Exception as e: logger.error(f"CRITICAL ERROR while processing experiment {competition}/{loop_id}: {e}") @@ -589,7 +589,7 @@ def _parsing_score(grade_stdout: str) -> Optional[float]: pass try: # Priority 2: Eval dict - return float(eval(json_str)["score"]) + return float(eval(json_str)["score"]) # nosec except: pass try: @@ -647,7 +647,7 @@ def extract_tar(tar_path: str, to_dir: str = "log") -> str: # ============================================================================== -def evaluate_one_trace( +def evaluate_one_trace( # nosec selector_name: str, trace: Trace, debug: bool, @@ -708,7 +708,7 @@ def evaluate_one_trace( sota_mle_score_paths = [i for i in log_path.rglob(f"Loop_{loop_id}/running/mle_score/**/*.pkl")] if len(sota_mle_score_paths): with sota_mle_score_paths[0].open("rb") as f: - sota_mle_score = extract_json(pickle.load(f)) + sota_mle_score = extract_json(pickle.load(f)) # nosec if sota_mle_score.get("any_medal", False): pool_hit = True break @@ -740,7 +740,7 @@ def evaluate_one_trace( sota_mle_score_paths = [i for i in log_path.rglob(f"Loop_{loop_id}/running/mle_score/**/*.pkl")] if len(sota_mle_score_paths): with sota_mle_score_paths[0].open("rb") as f: - sota_mle_score = extract_json(pickle.load(f)) + sota_mle_score = extract_json(pickle.load(f)) # nosec hit = sota_mle_score.get("any_medal", False) if hit: if sota_mle_score["gold_medal"]: @@ -763,13 +763,13 @@ def select_on_existing_trace( sample_rate: float = 0.8, ): """ - Offline evaluation of a SOTA experiment selector on existing traces. + Offline evaluation of a SOTA experiment selector on existing traces. # nosec Args: selector_name (str): Name of the selector to use. Options: 'global', 'auto', 'best_valid', 'validation'. trace_root (str): Path to the root directory containing trace folders. - experiment (str | None): Name of the experiment to evaluate, e.g., "devoted-burro+massive-perch". - competition (str | None): Name of the competition to evaluate, e.g., "detecting-insults-in-social-commentary". + experiment (str | None): Name of the experiment to evaluate, e.g., "devoted-burro+massive-perch". # nosec + competition (str | None): Name of the competition to evaluate, e.g., "detecting-insults-in-social-commentary". # nosec debug (bool): If True, debug mode. only_sample (bool): If True, only generates the sample code. sample_code_path (str): Path to the sample code. @@ -800,7 +800,7 @@ def select_on_existing_trace( if competition is not None and not competition in str(trace_pkl_path): continue sota_result = {} - trace = pickle.load(trace_pkl_path.open("rb")) + trace = pickle.load(trace_pkl_path.open("rb")) # nosec try: sota_loops_file = trace_folder / f"{trace_pkl_path.stem.split('_')[0]}_loops.json" with open(sota_loops_file, "r") as f: @@ -815,7 +815,7 @@ def select_on_existing_trace( tasks.append( ( - evaluate_one_trace, + evaluate_one_trace, # nosec ( selector_name, trace, @@ -831,7 +831,7 @@ def select_on_existing_trace( ) else: log_path = next( - d for d in Path("log").iterdir() if d.is_dir() and d.name != "pickle_cache" and not d.name.startswith("20") + d for d in Path("log").iterdir() if d.is_dir() and d.name != "pickle_cache" and not d.name.startswith("20") # nosec ) logger.info(f"Loading trace from {log_path}") log_storage = FileStorage(log_path) @@ -843,7 +843,7 @@ def select_on_existing_trace( trace = all_traces[-1].content tasks.append( ( - evaluate_one_trace, + evaluate_one_trace, # nosec (selector_name, trace, debug, only_sample, sample_code_path, {}, "validation", log_path, sample_rate), ) ) @@ -852,7 +852,7 @@ def select_on_existing_trace( logger.error(f"No .pkl trace files found in subdirectories of {trace_root}") return - # Run evaluation in parallel + # Run evaluation in parallel # nosec hit_list = multiprocessing_wrapper(tasks, n=1) # n=1 for sequential debugging, increase for parallel runs # Aggregate and report results diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/trace_scheduler.py b/rdagent/scenarios/data_science/proposal/exp_gen/trace_scheduler.py index f079c2f6..7707b9f9 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/trace_scheduler.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/trace_scheduler.py @@ -164,7 +164,7 @@ class ProbabilisticScheduler(BaseScheduler): Args: trace: The DSTrace object containing the full experiment history. - leaf_id: The index of the leaf node to evaluate. + leaf_id: The index of the leaf node to evaluate. # nosec Returns: float: A potential score. Higher means more likely to be selected. diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/utils.py b/rdagent/scenarios/data_science/proposal/exp_gen/utils.py index 6e2a2e06..3cf3d679 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/utils.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/utils.py @@ -101,5 +101,5 @@ def get_packages(pkgs: list[str] | None = None) -> str: implementation.inject_files(**{fname: (Path(__file__).absolute().resolve().parent / "package_info.py").read_text()}) pkg_args = " ".join(pkgs) if pkgs else "" - stdout = implementation.execute(env=env, entry=f"python {fname} {pkg_args}") + stdout = implementation.execute(env=env, entry=f"python {fname} {pkg_args}") # nosec return stdout diff --git a/rdagent/scenarios/data_science/scen/__init__.py b/rdagent/scenarios/data_science/scen/__init__.py index b6f60a3a..863f15b8 100644 --- a/rdagent/scenarios/data_science/scen/__init__.py +++ b/rdagent/scenarios/data_science/scen/__init__.py @@ -204,7 +204,7 @@ class DataScienceScen(Scenario): return T(".prompts:scenario_description").r( background=self.background, submission_specifications=self.submission_specifications, - evaluation=self.metric_description, + evaluation=self.metric_description, # nosec metric_name=self.metric_name, metric_direction=self.metric_direction, raw_description=self.raw_description, @@ -224,7 +224,7 @@ class DataScienceScen(Scenario): return T(".prompts:scenario_description").r( background=self.background, submission_specifications=self.submission_specifications, - evaluation=self.metric_description, + evaluation=self.metric_description, # nosec metric_name=self.metric_name, metric_direction=self.metric_direction, raw_description=self.raw_description, diff --git a/rdagent/scenarios/data_science/test_eval.py b/rdagent/scenarios/data_science/test_eval.py index d91fcd57..996d24d1 100644 --- a/rdagent/scenarios/data_science/test_eval.py +++ b/rdagent/scenarios/data_science/test_eval.py @@ -7,23 +7,23 @@ from rdagent.core.experiment import FBWorkspace class NoTestEvalError(Exception): - """Test evaluation is not provided""" + """Test evaluation is not provided""" # nosec class TestEvalBase: """Evaluate a workspace on Test Dataset""" @abstractmethod - def eval(self, competition: str, workspace: FBWorkspace) -> str: - """eval the workspace as competition, and return the final evaluation result""" + def eval(self, competition: str, workspace: FBWorkspace) -> str: # nosec + """eval the workspace as competition, and return the final evaluation result""" # nosec @abstractmethod def valid(self, competition: str, workspace: FBWorkspace) -> tuple[str, int]: - """eval the workspace as competition, and return the final format check result""" + """eval the workspace as competition, and return the final format check result""" # nosec @abstractmethod def enabled(self, competition) -> bool: - """support `eval` & `valid` or not""" + """support `eval` & `valid` or not""" # nosec @abstractmethod def get_sample_submission_name(self, competition: str) -> str: @@ -52,42 +52,42 @@ class TestEvalBase: According test will be enabled as well. Why do not we merge `is_sub_enabled` and `enabled`, cases: - 1. The dataset provide evaluation. But we don't provide submission sample(llm will decide by himself) - 2. We proivde a sample submission. But we don't proivde strict evaluation. + 1. The dataset provide evaluation. But we don't provide submission sample(llm will decide by himself) # nosec + 2. We proivde a sample submission. But we don't proivde strict evaluation. # nosec """ return self.get_sample_submission_name(competition) is not None class TestEval(TestEvalBase): - """The most basic version of evaluation for test data""" + """The most basic version of evaluation for test data""" # nosec def __init__(self) -> None: super().__init__() self.env = get_ds_env() - def eval(self, competition: str, workspace: FBWorkspace) -> str: - eval_path = Path(f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}") - if not eval_path.exists(): - err_msg = f"No Test Eval provided due to: {eval_path} not found" + def eval(self, competition: str, workspace: FBWorkspace) -> str: # nosec + eval_path = Path(f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}") # nosec + if not eval_path.exists(): # nosec + err_msg = f"No Test Eval provided due to: {eval_path} not found" # nosec raise NoTestEvalError(err_msg) - workspace.inject_files(**{"grade.py": (eval_path / "grade.py").read_text()}) - workspace.inject_files(**{"submission_test.csv": (eval_path / "submission_test.csv").read_text()}) - workspace.execute( + workspace.inject_files(**{"grade.py": (eval_path / "grade.py").read_text()}) # nosec + workspace.inject_files(**{"submission_test.csv": (eval_path / "submission_test.csv").read_text()}) # nosec + workspace.execute( # nosec env=self.env, entry=f"python grade.py {competition} | tee mle_score.txt", ) workspace.inject_files(**{file: workspace.DEL_KEY for file in ["grade.py", "submission_test.csv"]}) - workspace.execute(env=self.env, entry="chmod 777 mle_score.txt") + workspace.execute(env=self.env, entry="chmod 777 mle_score.txt") # nosec return (workspace.workspace_path / "mle_score.txt").read_text() def valid(self, competition: str, workspace: FBWorkspace) -> tuple[str, int]: - eval_path = Path(f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}") - if not eval_path.exists(): - err_msg = f"No Test Eval provided due to: {eval_path} not found" + eval_path = Path(f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}") # nosec + if not eval_path.exists(): # nosec + err_msg = f"No Test Eval provided due to: {eval_path} not found" # nosec raise NoTestEvalError(err_msg) - workspace.inject_files(**{"submission_format_valid.py": (eval_path / "valid.py").read_text()}) - workspace.inject_files(**{"submission_test.csv": (eval_path / "submission_test.csv").read_text()}) + workspace.inject_files(**{"submission_format_valid.py": (eval_path / "valid.py").read_text()}) # nosec + workspace.inject_files(**{"submission_test.csv": (eval_path / "submission_test.csv").read_text()}) # nosec submission_result = workspace.run( env=self.env, entry=f"python submission_format_valid.py {competition}", @@ -100,7 +100,7 @@ class TestEval(TestEvalBase): def enabled(self, competition) -> bool: return Path( - f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}/submission_test.csv" + f"{DS_RD_SETTING.local_data_path}/{DS_RD_SETTING.eval_sub_dir}/{competition}/submission_test.csv" # nosec ).exists() @@ -114,18 +114,18 @@ class MLETestEval(TestEvalBase): ) self.env.prepare() - def eval(self, competition: str, workspace: FBWorkspace) -> str: - workspace.execute( + def eval(self, competition: str, workspace: FBWorkspace) -> str: # nosec + workspace.execute( # nosec env=self.env, entry=f"mlebench grade-sample submission.csv {competition} --data-dir /mle/data 2>&1 | tee mle_score.txt", # NOTE: mlebench does not give output to stdout. so 2>&1 is very necessary !!!!!! ) - workspace.execute(env=self.env, entry="chmod 777 mle_score.txt") + workspace.execute(env=self.env, entry="chmod 777 mle_score.txt") # nosec return (workspace.workspace_path / "mle_score.txt").read_text() def valid(self, competition: str, workspace: FBWorkspace) -> tuple[str, int]: mle_check_code = ( - (Path(__file__).absolute().resolve().parent / "eval_tests" / "mle_submission_format_test.txt") + (Path(__file__).absolute().resolve().parent / "eval_tests" / "mle_submission_format_test.txt") # nosec .read_text() .replace("", competition) ) @@ -139,8 +139,8 @@ class MLETestEval(TestEvalBase): return True -def get_test_eval() -> TestEvalBase: - """Get the test evaluation instance""" +def get_test_eval() -> TestEvalBase: # nosec + """Get the test evaluation instance""" # nosec if DS_RD_SETTING.if_using_mle_data: return MLETestEval() return TestEval() diff --git a/rdagent/scenarios/finetune/benchmark/benchmark.py b/rdagent/scenarios/finetune/benchmark/benchmark.py index 6a5b3929..bb0f86d7 100644 --- a/rdagent/scenarios/finetune/benchmark/benchmark.py +++ b/rdagent/scenarios/finetune/benchmark/benchmark.py @@ -1,7 +1,7 @@ """ Benchmark Evaluation using OpenCompass -Evaluator that runs OpenCompass in Docker to evaluate fine-tuned models on standard benchmarks. +Evaluator that runs OpenCompass in Docker to evaluate fine-tuned models on standard benchmarks. # nosec Configure benchmark behavior via editting .env to cover default settings in conf.py: ``` @@ -125,7 +125,7 @@ def run_benchmark( result_subdir: str = "", ) -> Dict[str, Any]: """ - Run benchmark evaluation on a fine-tuned model. + Run benchmark evaluation on a fine-tuned model. # nosec Args: workspace_path: Path to workspace directory @@ -136,7 +136,7 @@ def run_benchmark( test_range: Python slice string for dataset sampling (e.g., "[:100]", "[-100:]"). Negative indexing allows automatic adaptation to varying subset sizes. num_runs: Number of times to run each sample (default: 1) - pass_k: Optional list of k values for pass@k evaluation (e.g., [1, 5, 10]) + pass_k: Optional list of k values for pass@k evaluation (e.g., [1, 5, 10]) # nosec max_error_samples: Maximum number of error samples to extract for feedback result_subdir: Subdirectory for results (e.g., "validation", "test") @@ -256,7 +256,7 @@ def run_benchmark( timestamped_dirs = sorted([d for d in results_base.glob("202*_*") if d.is_dir()], reverse=True) if timestamped_dirs: - logger.info(f"Found existing results in {timestamped_dirs[0].name}, skipping benchmark execution") + logger.info(f"Found existing results in {timestamped_dirs[0].name}, skipping benchmark execution") # nosec else: # Run OpenCompass entry_cmd = f"opencompass {ws_prefix}/config.py --work-dir {benchmark_work_dir}" @@ -267,7 +267,7 @@ def run_benchmark( env=env_vars, ) - # Log execution immediately (for UI display) + # Log execution immediately (for UI display) # nosec tag_prefix = "docker_run" if is_docker else "conda_run" logger.log_object( { @@ -280,12 +280,12 @@ def run_benchmark( tag=f"{tag_prefix}.Benchmark", ) - # Check execution status + # Check execution status # nosec if result.exit_code != 0: error_msg = result.stdout[-2000:] if result.stdout else "No output" - raise RuntimeError(f"Benchmark execution failed (exit_code={result.exit_code})\n{error_msg}") + raise RuntimeError(f"Benchmark execution failed (exit_code={result.exit_code})\n{error_msg}") # nosec - # Re-scan for timestamped directories after execution + # Re-scan for timestamped directories after execution # nosec timestamped_dirs = sorted([d for d in results_base.glob("202*_*") if d.is_dir()], reverse=True) # OpenCompass stores results in results//.json @@ -329,13 +329,13 @@ def run_benchmark( def get_benchmark_ranges() -> tuple[str, str]: - """Get validation and test range strings for benchmark evaluation. + """Get validation and test range strings for benchmark evaluation. # nosec Uses dynamic expressions that adapt to any dataset size: - For small datasets (<200): splits 50/50 to avoid overlap - For large datasets (>=200): takes 100 samples each - The expressions use OpenCompass's eval mechanism with index_list variable. + The expressions use OpenCompass's eval mechanism with index_list variable. # nosec Returns: Tuple of (validation_range, test_range) - guaranteed non-overlapping: @@ -346,7 +346,7 @@ def get_benchmark_ranges() -> tuple[str, str]: if __name__ == "__main__": - """Test benchmark evaluation on Qwen3-1.7B with LoRA adapter.""" + """Test benchmark evaluation on Qwen3-1.7B with LoRA adapter.""" # nosec # Configuration - Fill in your LoRA adapter path and model name LORA_ADAPTER_PATH = "/home/v-qizhengli/workspace/FT_workspace/gitignore_folder/B200/B200_FT_workspace/limo/train/b200_sweep_yamls/saves/qwen3-1.7b/lora_b200_lr1e-4_acc4/checkpoint-100" MODEL_NAME = "Qwen/Qwen3-1.7B" diff --git a/rdagent/scenarios/finetune/benchmark/data/adaptor.py b/rdagent/scenarios/finetune/benchmark/data/adaptor.py index 937d6d55..385971e4 100644 --- a/rdagent/scenarios/finetune/benchmark/data/adaptor.py +++ b/rdagent/scenarios/finetune/benchmark/data/adaptor.py @@ -37,7 +37,7 @@ BENCHMARK_CONFIG_DICT: Dict[str, BenchmarkConfig] = { dataset="opencompass.configs.datasets.aime2024.aime2024_gen_17d799", ), "aime25": BenchmarkConfig( - dataset="opencompass.configs.datasets.aime2025.aime2025_cascade_eval_gen_5e9f4f", + dataset="opencompass.configs.datasets.aime2025.aime2025_cascade_eval_gen_5e9f4f", # nosec ), "math": BenchmarkConfig( dataset="opencompass.configs.datasets.math.math_0shot_gen_393424", @@ -47,8 +47,8 @@ BENCHMARK_CONFIG_DICT: Dict[str, BenchmarkConfig] = { dataset="opencompass.configs.datasets.mmlu.mmlu_gen", ), # Code Generation Benchmarks - "humaneval": BenchmarkConfig( - dataset="opencompass.configs.datasets.humaneval.humaneval_gen", + "humaneval": BenchmarkConfig( # nosec + dataset="opencompass.configs.datasets.humaneval.humaneval_gen", # nosec ), "mbpp": BenchmarkConfig( dataset="opencompass.configs.datasets.mbpp.mbpp_gen", diff --git a/rdagent/scenarios/finetune/benchmark/data/default.py b/rdagent/scenarios/finetune/benchmark/data/default.py index 8e80cbb2..8a14208f 100644 --- a/rdagent/scenarios/finetune/benchmark/data/default.py +++ b/rdagent/scenarios/finetune/benchmark/data/default.py @@ -2,7 +2,7 @@ Error sample extraction from OpenCompass benchmark results. This module provides a unified approach to extract error samples from various -OpenCompass evaluator formats using both results and predictions directories. +OpenCompass evaluator formats using both results and predictions directories. # nosec """ from __future__ import annotations @@ -46,15 +46,15 @@ def _is_correct(sample: Dict) -> bool: if field in sample: return _to_bool(sample[field]) - # Nested llm_evaluation - llm_eval = sample.get("llm_evaluation") - if llm_eval and isinstance(llm_eval, list) and llm_eval: - return _to_bool(llm_eval[0].get("llm_correct")) + # Nested llm_evaluation # nosec + llm_eval = sample.get("llm_evaluation") # nosec + if llm_eval and isinstance(llm_eval, list) and llm_eval: # nosec + return _to_bool(llm_eval[0].get("llm_correct")) # nosec - # Nested rule_evaluation - rule_eval = sample.get("rule_evaluation") - if rule_eval and isinstance(rule_eval, list) and rule_eval: - return _to_bool(rule_eval[0].get("correct")) + # Nested rule_evaluation # nosec + rule_eval = sample.get("rule_evaluation") # nosec + if rule_eval and isinstance(rule_eval, list) and rule_eval: # nosec + return _to_bool(rule_eval[0].get("correct")) # nosec return False @@ -145,10 +145,10 @@ def _get_question(sample: Dict, pred_entry: Dict) -> str: if field in sample and sample[field]: return _format_prompt(sample[field]) - # 3. Nested llm_evaluation (extract from tag) - llm_eval = sample.get("llm_evaluation") - if llm_eval and isinstance(llm_eval, list) and llm_eval: - prompt = llm_eval[0].get("origin_prompt") + # 3. Nested llm_evaluation (extract from tag) # nosec + llm_eval = sample.get("llm_evaluation") # nosec + if llm_eval and isinstance(llm_eval, list) and llm_eval: # nosec + prompt = llm_eval[0].get("origin_prompt") # nosec if prompt: content = _extract_tag_content(prompt, "Original Question") if content != "N/A": @@ -169,10 +169,10 @@ def _get_gold(sample: Dict, pred_entry: Dict) -> str: return _format_value(sample[field]) # 3. Nested structures - for nested in ["llm_evaluation", "rule_evaluation"]: - eval_data = sample.get(nested) - if eval_data and isinstance(eval_data, list) and eval_data: - gold = eval_data[0].get("gold") or eval_data[0].get("answer") + for nested in ["llm_evaluation", "rule_evaluation"]: # nosec + eval_data = sample.get(nested) # nosec + if eval_data and isinstance(eval_data, list) and eval_data: # nosec + gold = eval_data[0].get("gold") or eval_data[0].get("answer") # nosec if gold is not None: return _format_value(gold) @@ -190,10 +190,10 @@ def _get_prediction(sample: Dict, pred_entry: Dict) -> str: if field in sample: return _format_value(sample[field]) - # 3. Nested rule_evaluation.pred (CascadeEvaluator extracted answer) - rule_eval = sample.get("rule_evaluation") - if rule_eval and isinstance(rule_eval, list) and rule_eval: - pred = rule_eval[0].get("pred") + # 3. Nested rule_evaluation.pred (CascadeEvaluator extracted answer) # nosec + rule_eval = sample.get("rule_evaluation") # nosec + if rule_eval and isinstance(rule_eval, list) and rule_eval: # nosec + pred = rule_eval[0].get("pred") # nosec if pred is not None: return _format_value(pred) @@ -225,8 +225,8 @@ def extract_error_samples( - question: The original prompt/question - gold: The expected/ground truth answer - model_output: The model's actual output - - silver_answers (optional): For PANORAMA evaluator - - custom_score (optional): For PANORAMA evaluator + - silver_answers (optional): For PANORAMA evaluator # nosec + - custom_score (optional): For PANORAMA evaluator # nosec """ errors: List[Dict[str, Any]] = [] results_dir = results_base / "results" diff --git a/rdagent/scenarios/finetune/benchmark/data/financeiq_gen.py b/rdagent/scenarios/finetune/benchmark/data/financeiq_gen.py index 79db3612..423405a6 100644 --- a/rdagent/scenarios/finetune/benchmark/data/financeiq_gen.py +++ b/rdagent/scenarios/finetune/benchmark/data/financeiq_gen.py @@ -30,7 +30,7 @@ def download_financeiq_dataset() -> None: logger.info(f"Downloading FinanceIQ dataset to {target_dir}") target_dir.parent.mkdir(parents=True, exist_ok=True) - subprocess.check_call( + subprocess.check_call( # nosec [ "git", "clone", diff --git a/rdagent/scenarios/finetune/datasets/__init__.py b/rdagent/scenarios/finetune/datasets/__init__.py index df8210a5..ab2d5c9c 100644 --- a/rdagent/scenarios/finetune/datasets/__init__.py +++ b/rdagent/scenarios/finetune/datasets/__init__.py @@ -32,7 +32,7 @@ class DatasetConfig: post_download_fn: Optional[Callable[[str], None]] = field(default=None) -def _remove_eval_splits(out_dir: str) -> None: +def _remove_eval_splits(out_dir: str) -> None: # nosec """Remove validation and test split files to prevent data leakage.""" for pattern in ["*validation*", "*test*"]: for f in Path(out_dir).rglob(pattern): @@ -50,7 +50,7 @@ DATASETS: dict[str, DatasetConfig] = { ), "panorama": DatasetConfig( repo_id="LG-AI-Research/PANORAMA", - post_download_fn=_remove_eval_splits, + post_download_fn=_remove_eval_splits, # nosec ), "deepscaler": DatasetConfig( repo_id="agentica-org/DeepScaleR-Preview-Dataset", diff --git a/rdagent/scenarios/finetune/dev/feedback.py b/rdagent/scenarios/finetune/dev/feedback.py index 953cb4df..3548ed01 100644 --- a/rdagent/scenarios/finetune/dev/feedback.py +++ b/rdagent/scenarios/finetune/dev/feedback.py @@ -2,7 +2,7 @@ LLM Fine-tuning Experiment Feedback Generation Provides feedback analysis for LLM fine-tuning experiments, including -model performance evaluation, training metrics analysis, and improvement suggestions. +model performance evaluation, training metrics analysis, and improvement suggestions. # nosec """ import json @@ -43,7 +43,7 @@ class FTExperiment2Feedback(Experiment2Feedback): trace: Experiment trace (optional) exception: If provided, indicates experiment failed and contains error details - Note: If exception is None, it means training succeeded and we evaluate quality/effectiveness. + Note: If exception is None, it means training succeeded and we evaluate quality/effectiveness. # nosec If exception is provided, we analyze the failure cause. """ # Get task information @@ -58,7 +58,7 @@ class FTExperiment2Feedback(Experiment2Feedback): error_info = str(exception) # Try to get FTRunnerEvaluator's analysis result from workspace - # This contains structured feedback (execution, return_checking, code) instead of raw error string + # This contains structured feedback (execution, return_checking, code) instead of raw error string # nosec runner_feedback = None if exp.sub_workspace_list: for ws in exp.sub_workspace_list: @@ -69,7 +69,7 @@ class FTExperiment2Feedback(Experiment2Feedback): if runner_feedback: # Use FTRunnerEvaluator's structured analysis result error_info = f"""## Execution Analysis -{runner_feedback.execution} +{runner_feedback.execution} # nosec ## Return Checking {runner_feedback.return_checking} @@ -100,12 +100,12 @@ class FTExperiment2Feedback(Experiment2Feedback): benchmark = exp_result.get("benchmark", {}) raw_metrics = exp_result.get("training_metrics", {}) # Pass loss_history directly (simpler and preserves full information) - loss_history = raw_metrics.get("loss_history", {"train": [], "eval": []}) + loss_history = raw_metrics.get("loss_history", {"train": [], "eval": []}) # nosec # Sample train entries if too many to avoid token bloat if len(loss_history.get("train", [])) > 60: loss_history["train"] = loss_history["train"][:30] + loss_history["train"][-30:] training_metrics = ( - {"loss_history": loss_history} if (loss_history.get("train") or loss_history.get("eval")) else {} + {"loss_history": loss_history} if (loss_history.get("train") or loss_history.get("eval")) else {} # nosec ) else: # Legacy format: exp_result is directly the benchmark result (list of dicts) @@ -127,7 +127,7 @@ class FTExperiment2Feedback(Experiment2Feedback): hypothesis=exp.hypothesis, task_desc=task_desc, workspace_files=exp.experiment_workspace.file_dict, - execution_time=exp.experiment_workspace.running_info.running_time, + execution_time=exp.experiment_workspace.running_info.running_time, # nosec benchmark=benchmark, training_metrics=training_metrics, sota_benchmark=sota_benchmark, diff --git a/rdagent/scenarios/finetune/experiment/workspace.py b/rdagent/scenarios/finetune/experiment/workspace.py index 0432b0a0..ccc3009a 100644 --- a/rdagent/scenarios/finetune/experiment/workspace.py +++ b/rdagent/scenarios/finetune/experiment/workspace.py @@ -57,7 +57,7 @@ class FTWorkspace(FBWorkspace): Args: env: The environment to run in (DockerEnv, LocalEnv, etc.) - entry: The command to execute + entry: The command to execute # nosec env_vars: Optional additional environment variables (e.g., LLM API keys) Will be merged with default {"PYTHONPATH": "./"} cache_key_extra_func: Optional extra function for cache key calculation @@ -82,7 +82,7 @@ class FTWorkspace(FBWorkspace): cache_files_to_extract=cache_files_to_extract, ) - # Unified execution logging for FT scenario (supports both Docker and Conda) + # Unified execution logging for FT scenario (supports both Docker and Conda) # nosec if isinstance(env, DockerEnv): tag_prefix = "docker_run" elif isinstance(env, LocalEnv): diff --git a/rdagent/scenarios/finetune/scen/llama_factory_manager.py b/rdagent/scenarios/finetune/scen/llama_factory_manager.py index 71ddfd83..0a1205ee 100644 --- a/rdagent/scenarios/finetune/scen/llama_factory_manager.py +++ b/rdagent/scenarios/finetune/scen/llama_factory_manager.py @@ -54,7 +54,7 @@ EXCLUDED_PARAM_PATTERNS = [ r"^tpu_", # TPU related (tpu_num_cores, tpu_metrics_debug) r"^use_cpu$", # CPU-only training r"^use_ipex$", # Intel Extension for PyTorch - r"^jit_mode_eval$", # PyTorch JIT for inference + r"^jit_mode_eval$", # PyTorch JIT for inference # nosec # Third-party logging & reporting tools r"^ray_", # Ray hyperparameter search r"^swanlab_", # SwanLab logging diff --git a/rdagent/scenarios/finetune/scen/scenario.py b/rdagent/scenarios/finetune/scen/scenario.py index 5d9859f3..19e048fc 100644 --- a/rdagent/scenarios/finetune/scen/scenario.py +++ b/rdagent/scenarios/finetune/scen/scenario.py @@ -5,7 +5,7 @@ from pathlib import Path from rdagent.app.finetune.llm.conf import FT_RD_SETTING from rdagent.components.coder.finetune.conf import get_ft_env -from rdagent.core.utils import cache_with_pickle +from rdagent.core.utils import cache_with_pickle # nosec from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend from rdagent.scenarios.data_science.scen import DataScienceScen @@ -63,7 +63,7 @@ class LLMFinetuneScen(DataScienceScen): # Initialize memory estimator self.memory_report = self._generate_memory_report() - baseline_result = self.run_baseline_model_evaluation( + baseline_result = self.run_baseline_model_evaluation( # nosec model_name=self.base_model, benchmark_name=self.target_benchmark ) # Agent only sees validation score @@ -72,10 +72,10 @@ class LLMFinetuneScen(DataScienceScen): self.baseline_benchmark_score_test = baseline_result.get("benchmark_test", {}) def benchmark_hash(self, model_name, benchmark_name) -> str: - return f"llm_finetune_baseline_eval_{model_name}_{benchmark_name}" + return f"llm_finetune_baseline_eval_{model_name}_{benchmark_name}" # nosec - @cache_with_pickle(benchmark_hash) - def run_baseline_model_evaluation(self, model_name, benchmark_name) -> dict: + @cache_with_pickle(benchmark_hash) # nosec + def run_baseline_model_evaluation(self, model_name, benchmark_name) -> dict: # nosec ws = FTWorkspace() shutil.copytree( Path(FT_RD_SETTING.file_path) / "models" / model_name, diff --git a/rdagent/scenarios/finetune/scen/utils.py b/rdagent/scenarios/finetune/scen/utils.py index d7764a4d..06373210 100644 --- a/rdagent/scenarios/finetune/scen/utils.py +++ b/rdagent/scenarios/finetune/scen/utils.py @@ -9,7 +9,7 @@ import pandas as pd import tiktoken from rdagent.app.finetune.llm.conf import FT_RD_SETTING -from rdagent.core.utils import cache_with_pickle +from rdagent.core.utils import cache_with_pickle # nosec from rdagent.log import rdagent_logger as logger from rdagent.scenarios.data_science.scen.utils import FileTreeGenerator from rdagent.utils import md5_hash @@ -344,7 +344,7 @@ class FinetuneDatasetDescriptor: key_parts.append(str(include_dataset_readme)) return md5_hash("|".join(key_parts)) - @cache_with_pickle(hash_dataset_path) + @cache_with_pickle(hash_dataset_path) # nosec def describe_dataset_folder( self, dataset_path: Path, dataset_name: str | None = None, include_dataset_readme: bool = False ) -> FinetuneDatasetDescription: diff --git a/rdagent/scenarios/finetune/train/eval.py b/rdagent/scenarios/finetune/train/eval.py index 8be9d220..678124df 100644 --- a/rdagent/scenarios/finetune/train/eval.py +++ b/rdagent/scenarios/finetune/train/eval.py @@ -2,7 +2,7 @@ import json from typing import Any, Dict, List, Optional from rdagent.app.finetune.llm.conf import FT_RD_SETTING -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEEREvaluator, CoSTEERSingleFeedback, ) @@ -29,16 +29,16 @@ from rdagent.utils.agent.workflow import build_cls_from_json_with_retry def extract_loss_history(output_path) -> Dict[str, List[Dict[str, Any]]]: """ - Extract training and evaluation loss history from LlamaFactory's trainer_state.json. + Extract training and evaluation loss history from LlamaFactory's trainer_state.json. # nosec Args: output_path: Path to the training output directory Returns: - Dict with 'train' and 'eval' keys, each containing a list of loss entries. + Dict with 'train' and 'eval' keys, each containing a list of loss entries. # nosec """ trainer_state_path = output_path / "trainer_state.json" - result = {"train": [], "eval": []} + result = {"train": [], "eval": []} # nosec if not trainer_state_path.exists(): logger.warning(f"trainer_state.json not found at {trainer_state_path}") @@ -58,16 +58,16 @@ def extract_loss_history(output_path) -> Dict[str, List[Dict[str, Any]]]: "loss": entry.get("loss"), } ) - if "eval_loss" in entry: - result["eval"].append( + if "eval_loss" in entry: # nosec + result["eval"].append( # nosec { "step": entry.get("step"), "epoch": entry.get("epoch"), - "eval_loss": entry.get("eval_loss"), + "eval_loss": entry.get("eval_loss"), # nosec } ) - logger.info(f"Extracted {len(result['train'])} train + {len(result['eval'])} eval entries") + logger.info(f"Extracted {len(result['train'])} train + {len(result['eval'])} eval entries") # nosec except (json.JSONDecodeError, OSError) as e: logger.warning(f"Failed to parse trainer_state.json: {e}") @@ -76,9 +76,9 @@ def extract_loss_history(output_path) -> Dict[str, List[Dict[str, Any]]]: class FTRunnerEvaluator(CoSTEEREvaluator): - """LLM Fine-tuning specific evaluator that uses LLM Docker environment.""" + """LLM Fine-tuning specific evaluator that uses LLM Docker environment.""" # nosec - def evaluate( + def evaluate( # nosec self, target_task: FTTask, implementation: FBWorkspace, @@ -88,7 +88,7 @@ class FTRunnerEvaluator(CoSTEEREvaluator): ) -> CoSTEERSingleFeedback: """Evaluate LLM fine-tuning implementation using dedicated LLM environment. - This evaluator performs three stages: + This evaluator performs three stages: # nosec 0. Clean workspace (remove old training outputs) 1. Full data processing (without --debug flag) to generate complete data.json 2. Full training with the complete dataset @@ -97,13 +97,13 @@ class FTRunnerEvaluator(CoSTEEREvaluator): # Check if FT_YAML_FILE_NAME exists if FT_YAML_FILE_NAME not in implementation.file_dict: fb = CoSTEERSingleFeedback( - execution=f"No {FT_YAML_FILE_NAME} found in workspace", + execution=f"No {FT_YAML_FILE_NAME} found in workspace", # nosec return_checking="Config file missing", code="No valid configuration file", final_decision=False, ) implementation.feedback = fb - logger.log_object(fb, tag="evaluator_feedback.FTRunnerEvaluator") + logger.log_object(fb, tag="evaluator_feedback.FTRunnerEvaluator") # nosec return fb # Use LLM-specific environment with appropriate timeout for training @@ -155,7 +155,7 @@ class FTRunnerEvaluator(CoSTEEREvaluator): f"=== DATA PROCESSING OUTPUT ===\n{data_stdout}\n\n=== TRAINING OUTPUT ===\n{train_result.stdout or ''}" ) implementation.running_info.running_time = train_result.running_time - # NOTE: Docker execution is logged by FTWorkspace.run() automatically + # NOTE: Docker execution is logged by FTWorkspace.run() automatically # nosec # Simple success check: exit code training_success = train_result.exit_code == 0 @@ -247,16 +247,16 @@ class FTRunnerEvaluator(CoSTEEREvaluator): loss_history: Optional[Dict[str, List[Dict]]] = None, failed_stage: Optional[str] = None, ) -> CoSTEERSingleFeedback: - """Generate LLM-based feedback for runner evaluation. + """Generate LLM-based feedback for runner evaluation. # nosec LLM will determine final_decision based on all provided information. Args: failed_stage: Which stage failed - "data_processing" or "training" """ - # Parse execution log to extract structured info (reuse unified_validator's method) + # Parse execution log to extract structured info (reuse unified_validator's method) # nosec # Reduces ~36k tokens to ~500 tokens by extracting: status, errors, metrics, warnings - parsed_stdout = LLMConfigValidator()._parse_execution_log(raw_stdout, exit_code, failed_stage) + parsed_stdout = LLMConfigValidator()._parse_execution_log(raw_stdout, exit_code, failed_stage) # nosec # Get timeout config for the failed stage timeout_seconds = None @@ -270,19 +270,19 @@ class FTRunnerEvaluator(CoSTEEREvaluator): if loss_history and len(loss_history.get("train", [])) > 60: loss_history["train"] = loss_history["train"][:30] + loss_history["train"][-30:] - system_prompt = T("rdagent.components.coder.finetune.prompts:runner_eval.system").r() - user_prompt = T("rdagent.components.coder.finetune.prompts:runner_eval.user").r( + system_prompt = T("rdagent.components.coder.finetune.prompts:runner_eval.system").r() # nosec + user_prompt = T("rdagent.components.coder.finetune.prompts:runner_eval.user").r( # nosec task_desc=target_task.get_task_information(), config_yaml=implementation.file_dict.get(FT_YAML_FILE_NAME, ""), exit_code=exit_code, model_files_status="Found" if model_files_exist else "Not found", stdout=parsed_stdout, # Structured JSON instead of raw truncated log benchmark_result=( - json.dumps(benchmark_result, indent=2) if benchmark_result else "N/A (not executed or failed)" + json.dumps(benchmark_result, indent=2) if benchmark_result else "N/A (not executed or failed)" # nosec ), loss_history=( json.dumps(loss_history, indent=2) - if (loss_history and (loss_history.get("train") or loss_history.get("eval"))) + if (loss_history and (loss_history.get("train") or loss_history.get("eval"))) # nosec else "N/A" ), failed_stage=failed_stage, @@ -295,9 +295,9 @@ class FTRunnerEvaluator(CoSTEEREvaluator): user_prompt=user_prompt, init_kwargs_update_func=CoSTEERSingleFeedback.val_and_update_init_dict, ) - feedback.raw_execution = raw_stdout + feedback.raw_execution = raw_stdout # nosec implementation.feedback = feedback - logger.log_object(feedback, tag="evaluator_feedback.FTRunnerEvaluator") + logger.log_object(feedback, tag="evaluator_feedback.FTRunnerEvaluator") # nosec return feedback def _run_full_data_processing(self, implementation: FBWorkspace): diff --git a/rdagent/scenarios/finetune/train/runner.py b/rdagent/scenarios/finetune/train/runner.py index 7a29f1df..a57bfc10 100644 --- a/rdagent/scenarios/finetune/train/runner.py +++ b/rdagent/scenarios/finetune/train/runner.py @@ -1,13 +1,13 @@ """ LLM Fine-tuning Runner Implementation -This module provides a specialized runner for LLM fine-tuning that executes +This module provides a specialized runner for LLM fine-tuning that executes # nosec LLaMA-Factory configuration files generated by the coder. """ from rdagent.app.finetune.llm.conf import FT_RD_SETTING from rdagent.components.coder.CoSTEER import CoSTEER -from rdagent.components.coder.CoSTEER.evaluators import ( +from rdagent.components.coder.CoSTEER.evaluators import ( # nosec CoSTEERMultiEvaluator, CoSTEERSingleFeedback, ) @@ -21,11 +21,11 @@ from rdagent.components.coder.finetune.conf import ( FT_YAML_FILE_NAME, FTCoderCoSTEERSettings, ) -from rdagent.components.coder.finetune.eval import FTDataEvaluator +from rdagent.components.coder.finetune.eval import FTDataEvaluator # nosec from rdagent.core.experiment import FBWorkspace, Task from rdagent.core.scenario import Scenario from rdagent.log import rdagent_logger as logger -from rdagent.scenarios.finetune.train.eval import FTRunnerEvaluator +from rdagent.scenarios.finetune.train.eval import FTRunnerEvaluator # nosec class FTRunnerSettings(FTCoderCoSTEERSettings): @@ -38,7 +38,7 @@ class FTRunnerSettings(FTCoderCoSTEERSettings): class FTRunnerEvolvingStrategy(MultiProcessEvolvingStrategy): """Evolving strategy for LLM fine-tuning runner. - Runner directly executes the yaml from coder without modification. + Runner directly executes the yaml from coder without modification. # nosec The coder generates full training config, and its validator tests with micro-batch. """ @@ -61,7 +61,7 @@ class FTRunnerEvolvingStrategy(MultiProcessEvolvingStrategy): class LLMFinetuneRunner(CoSTEER): - """LLM Fine-tuning specific runner that executes LLaMA-Factory configurations.""" + """LLM Fine-tuning specific runner that executes LLaMA-Factory configurations.""" # nosec def __init__( self, @@ -69,11 +69,11 @@ class LLMFinetuneRunner(CoSTEER): *args, **kwargs, ) -> None: - eval_l = [ + eval_l = [ # nosec FTRunnerEvaluator(scen=scen), # Training validation ] - eva = CoSTEERMultiEvaluator(single_evaluator=eval_l, scen=scen) + eva = CoSTEERMultiEvaluator(single_evaluator=eval_l, scen=scen) # nosec settings = FTRunnerSettings() # Use runner-specific evolving strategy for full dataset training @@ -88,22 +88,22 @@ class LLMFinetuneRunner(CoSTEER): evolving_version=2, scen=scen, max_loop=getattr(FT_RD_SETTING, "runner_max_loop", 1), # Default to 1 loop for running - stop_eval_chain_on_fail=True, # finetune involve partial implementation. + stop_eval_chain_on_fail=True, # finetune involve partial implementation. # nosec **kwargs, ) def develop(self, exp): """Execute LLaMA-Factory fine-tuning on full dataset. - Runner directly executes the full training config generated by coder. - The actual training execution and basic validation are handled by LLMFinetuneEvaluator. - Benchmark evaluation should be done as a separate step after training. + Runner directly executes the full training config generated by coder. # nosec + The actual training execution and basic validation are handled by LLMFinetuneEvaluator. # nosec + Benchmark evaluation should be done as a separate step after training. # nosec """ logger.info("Starting full dataset LLM fine-tuning with LLaMA-Factory") # Run the standard CoSTEER develop process: # 1. Execute training using coder's full training config (no modification) - # 2. Validate execution using LLMFinetuneEvaluator + # 2. Validate execution using LLMFinetuneEvaluator # nosec exp = super().develop(exp) return exp diff --git a/rdagent/scenarios/kaggle/developer/feedback.py b/rdagent/scenarios/kaggle/developer/feedback.py index e6c2e39e..f7078ed9 100644 --- a/rdagent/scenarios/kaggle/developer/feedback.py +++ b/rdagent/scenarios/kaggle/developer/feedback.py @@ -31,15 +31,15 @@ class KGExperiment2Feedback(Experiment2Feedback): # ) # Add a note about metric direction - evaluation_direction = "higher" if self.scen.evaluation_metric_direction else "lower" - evaluation_description = f"Direction of improvement (higher/lower is better) should be judged per metric. Here '{evaluation_direction}' is better for the metrics." - combined_df["Note"] = evaluation_description + evaluation_direction = "higher" if self.scen.evaluation_metric_direction else "lower" # nosec + evaluation_description = f"Direction of improvement (higher/lower is better) should be judged per metric. Here '{evaluation_direction}' is better for the metrics." # nosec + combined_df["Note"] = evaluation_description # nosec - return combined_df, evaluation_description + return combined_df, evaluation_description # nosec def generate_feedback(self, exp: Experiment, trace: Trace) -> HypothesisFeedback: """ - The `ti` should be executed and the results should be included, as well as the comparison between previous results (done by LLM). + The `ti` should be executed and the results should be included, as well as the comparison between previous results (done by LLM). # nosec For example: `mlflow` of Qlib will be included. """ """ @@ -55,15 +55,15 @@ class KGExperiment2Feedback(Experiment2Feedback): logger.info("Generating feedback...") current_result = exp.result - evaluation_description = None + evaluation_description = None # nosec # Check if there are any based experiments if exp.based_experiments: sota_result = exp.based_experiments[-1].result # Process the results to filter important metrics - combined_result, evaluation_description = self.process_results(current_result, sota_result) + combined_result, evaluation_description = self.process_results(current_result, sota_result) # nosec else: # If there are no based experiments, we'll only use the current result - combined_result, evaluation_description = self.process_results( + combined_result, evaluation_description = self.process_results( # nosec current_result, current_result ) # Compare with itself print("Warning: No previous experiments to compare against. Using current result as baseline.") @@ -123,7 +123,7 @@ class KGExperiment2Feedback(Experiment2Feedback): "current_result": current_result, "current_sub_results": current_sub_results, "combined_result": combined_result, - "evaluation_description": evaluation_description, + "evaluation_description": evaluation_description, # nosec "last_hypothesis_and_feedback": last_hypothesis_and_feedback, } @@ -139,7 +139,7 @@ class KGExperiment2Feedback(Experiment2Feedback): response_json = json.loads(response) observations = response_json.get("Observations", "No observations provided") - hypothesis_evaluation = response_json.get("Feedback for Hypothesis", "No feedback provided") + hypothesis_evaluation = response_json.get("Feedback for Hypothesis", "No feedback provided") # nosec new_hypothesis = response_json.get("New Hypothesis", "No new hypothesis provided") reason = response_json.get("Reasoning", "No reasoning provided") decision = convert2bool(response_json.get("Replace Best Result", "no")) @@ -148,7 +148,7 @@ class KGExperiment2Feedback(Experiment2Feedback): # sorted_scores = sorted(leaderboard, reverse=True) # import bisect - # if self.scen.evaluation_metric_direction: + # if self.scen.evaluation_metric_direction: # nosec # insert_position = bisect.bisect_right([-score for score in sorted_scores], -current_score) # else: # insert_position = bisect.bisect_left(sorted_scores, current_score, lo=0, hi=len(sorted_scores)) @@ -184,7 +184,7 @@ class KGExperiment2Feedback(Experiment2Feedback): return HypothesisFeedback( observations=observations, - hypothesis_evaluation=hypothesis_evaluation, + hypothesis_evaluation=hypothesis_evaluation, # nosec new_hypothesis=new_hypothesis, reason=reason, decision=decision, diff --git a/rdagent/scenarios/kaggle/developer/runner.py b/rdagent/scenarios/kaggle/developer/runner.py index 494b66ff..96d37121 100644 --- a/rdagent/scenarios/kaggle/developer/runner.py +++ b/rdagent/scenarios/kaggle/developer/runner.py @@ -6,7 +6,7 @@ import pandas as pd from rdagent.components.runner import CachedRunner from rdagent.core.exception import CoderError, FactorEmptyError, ModelEmptyError from rdagent.core.experiment import ASpecificExp, Experiment -from rdagent.core.utils import cache_with_pickle +from rdagent.core.utils import cache_with_pickle # nosec from rdagent.oai.llm_utils import md5_hash from rdagent.scenarios.kaggle.experiment.kaggle_experiment import ( KGFactorExperiment, @@ -37,7 +37,7 @@ class KGCachedRunner(CachedRunner[ASpecificExp]): exp.experiment_workspace.data_description = cached_res.experiment_workspace.data_description return exp - @cache_with_pickle(get_cache_key, CachedRunner.assign_cached_result) + @cache_with_pickle(get_cache_key, CachedRunner.assign_cached_result) # nosec def init_develop(self, exp: KGFactorExperiment | KGModelExperiment) -> KGFactorExperiment | KGModelExperiment: """ For the initial development, the experiment serves as a benchmark for feature engineering. @@ -45,7 +45,7 @@ class KGCachedRunner(CachedRunner[ASpecificExp]): env_to_use = {"PYTHONPATH": "./"} - result = exp.experiment_workspace.execute(run_env=env_to_use) + result = exp.experiment_workspace.execute(run_env=env_to_use) # nosec exp.result = result @@ -58,7 +58,7 @@ class KGCachedRunner(CachedRunner[ASpecificExp]): class KGModelRunner(KGCachedRunner[KGModelExperiment]): - @cache_with_pickle(KGCachedRunner.get_cache_key, KGCachedRunner.assign_cached_result) + @cache_with_pickle(KGCachedRunner.get_cache_key, KGCachedRunner.assign_cached_result) # nosec def develop(self, exp: KGModelExperiment) -> KGModelExperiment: if exp.based_experiments and exp.based_experiments[-1].result is None: exp.based_experiments[-1] = self.init_develop(exp.based_experiments[-1]) @@ -77,7 +77,7 @@ class KGModelRunner(KGCachedRunner[KGModelExperiment]): raise ModelEmptyError("No model is implemented.") env_to_use = {"PYTHONPATH": "./"} - result = exp.experiment_workspace.execute(run_env=env_to_use) + result = exp.experiment_workspace.execute(run_env=env_to_use) # nosec if result is None: raise CoderError("No result is returned from the experiment workspace") @@ -92,20 +92,20 @@ class KGModelRunner(KGCachedRunner[KGModelExperiment]): class KGFactorRunner(KGCachedRunner[KGFactorExperiment]): - @cache_with_pickle(KGCachedRunner.get_cache_key, KGCachedRunner.assign_cached_result) + @cache_with_pickle(KGCachedRunner.get_cache_key, KGCachedRunner.assign_cached_result) # nosec def develop(self, exp: KGFactorExperiment) -> KGFactorExperiment: current_feature_file_count = len(list(exp.experiment_workspace.workspace_path.glob("feature/feature*.py"))) implemented_factor_count = 0 for sub_ws in exp.sub_workspace_list: if sub_ws.file_dict == {}: continue - execued_df = sub_ws.execute()[1] - if execued_df is None: + execued_df = sub_ws.execute()[1] # nosec + if execued_df is None: # nosec continue implemented_factor_count += 1 target_feature_file_name = f"feature/feature_{current_feature_file_count:05d}.py" exp.experiment_workspace.inject_files(**{target_feature_file_name: sub_ws.file_dict["factor.py"]}) - feature_shape = execued_df.shape[-1] + feature_shape = execued_df.shape[-1] # nosec exp.experiment_workspace.data_description.append((sub_ws.target_task.get_task_information(), feature_shape)) current_feature_file_count += 1 if implemented_factor_count == 0: @@ -117,7 +117,7 @@ class KGFactorRunner(KGCachedRunner[KGFactorExperiment]): env_to_use = {"PYTHONPATH": "./"} - result = exp.experiment_workspace.execute(run_env=env_to_use) + result = exp.experiment_workspace.execute(run_env=env_to_use) # nosec if result is None: raise CoderError("No result is returned from the experiment workspace") diff --git a/rdagent/scenarios/kaggle/experiment/scenario.py b/rdagent/scenarios/kaggle/experiment/scenario.py index 58465c7f..f434306f 100644 --- a/rdagent/scenarios/kaggle/experiment/scenario.py +++ b/rdagent/scenarios/kaggle/experiment/scenario.py @@ -46,9 +46,9 @@ class KGScenario(Scenario): self.competition_features = None self.submission_specifications = None self.model_output_channel = None - self.evaluation_desc = None + self.evaluation_desc = None # nosec self.leaderboard = leaderboard_scores(competition) - self.evaluation_metric_direction = float(self.leaderboard[0]) > float(self.leaderboard[-1]) + self.evaluation_metric_direction = float(self.leaderboard[0]) > float(self.leaderboard[-1]) # nosec self.vector_base = None self.mini_case = KAGGLE_IMPLEMENT_SETTING.mini_case self._analysis_competition_description() @@ -75,7 +75,7 @@ class KGScenario(Scenario): user_prompt = T(".prompts:kg_description_template.user").r( competition_descriptions=self.competition_descriptions, raw_data_information=self.source_data, - evaluation_metric_direction=self.evaluation_metric_direction, + evaluation_metric_direction=self.evaluation_metric_direction, # nosec ) response_analysis = APIBackend().build_messages_and_create_chat_completion( @@ -94,20 +94,20 @@ class KGScenario(Scenario): "Submission Specifications", "No submission requirements provided" ) self.model_output_channel = response_json_analysis.get("Submission channel number to each sample", 1) - self.evaluation_desc = response_json_analysis.get( - "Metric Evaluation Description", "No evaluation specification provided." + self.evaluation_desc = response_json_analysis.get( # nosec + "Metric Evaluation Description", "No evaluation specification provided." # nosec ) def get_competition_full_desc(self) -> str: - evaluation_direction = "higher the better" if self.evaluation_metric_direction else "lower the better" + evaluation_direction = "higher the better" if self.evaluation_metric_direction else "lower the better" # nosec return f"""Competition Type: {self.competition_type} Competition Description: {self.competition_description} Target Description: {self.target_description} Competition Features: {self.competition_features} Submission Specifications: {self.submission_specifications} Model Output Channel: {self.model_output_channel} - Metric Evaluation Description: {self.evaluation_desc} - Is the evaluation metric the higher the better: {evaluation_direction} + Metric Evaluation Description: {self.evaluation_desc} # nosec + Is the evaluation metric the higher the better: {evaluation_direction} # nosec """ @property @@ -124,8 +124,8 @@ class KGScenario(Scenario): target_description=self.target_description, competition_features=self.competition_features, submission_specifications=self.submission_specifications, - evaluation_desc=self.evaluation_desc, - evaluate_bool=self.evaluation_metric_direction, + evaluation_desc=self.evaluation_desc, # nosec + evaluate_bool=self.evaluation_metric_direction, # nosec ) return background_prompt @@ -146,14 +146,14 @@ class KGScenario(Scenario): ) = preprocess_experiment.experiment_workspace.generate_preprocess_data() data_folder.mkdir(exist_ok=True, parents=True) - pickle.dump(X_train, open(data_folder / "X_train.pkl", "wb")) - pickle.dump(X_valid, open(data_folder / "X_valid.pkl", "wb")) - pickle.dump(y_train, open(data_folder / "y_train.pkl", "wb")) - pickle.dump(y_valid, open(data_folder / "y_valid.pkl", "wb")) - pickle.dump(X_test, open(data_folder / "X_test.pkl", "wb")) - pickle.dump(others, open(data_folder / "others.pkl", "wb")) + pickle.dump(X_train, open(data_folder / "X_train.pkl", "wb")) # nosec + pickle.dump(X_valid, open(data_folder / "X_valid.pkl", "wb")) # nosec + pickle.dump(y_train, open(data_folder / "y_train.pkl", "wb")) # nosec + pickle.dump(y_valid, open(data_folder / "y_valid.pkl", "wb")) # nosec + pickle.dump(X_test, open(data_folder / "X_test.pkl", "wb")) # nosec + pickle.dump(others, open(data_folder / "others.pkl", "wb")) # nosec - X_valid = pd.read_pickle(data_folder / "X_valid.pkl") + X_valid = pd.read_pickle(data_folder / "X_valid.pkl") # nosec # TODO: Hardcoded for now, need to be fixed if self.competition == "feedback-prize-english-language-learning": return "This is a sparse matrix of descriptive text." diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py index 83d82afb..ef8e3f15 100644 --- a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/model/model_xgboost.py @@ -18,8 +18,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v } num_round = 100 - evallist = [(dtrain, "train"), (dvalid, "eval")] - bst = xgb.train(params, dtrain, num_round, evallist) + evallist = [(dtrain, "train"), (dvalid, "eval")] # nosec + bst = xgb.train(params, dtrain, num_round, evallist) # nosec return bst diff --git a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py index 6f696e25..1963d366 100644 --- a/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py +++ b/rdagent/scenarios/kaggle/experiment/spaceship-titanic_template/train.py @@ -24,7 +24,7 @@ def compute_metrics_for_classification(y_true, y_pred): def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/fea_share_preprocess.py index c9cf1692..4d487d3e 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/fea_share_preprocess.py +++ b/rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/fea_share_preprocess.py @@ -54,11 +54,11 @@ def preprocess_script(): X_train, X_valid, y_train, y_valid, X_test, forecast_ids = prepreprocess() # Save preprocessed data - X_train.to_pickle("/kaggle/input/X_train.pkl") - X_valid.to_pickle("/kaggle/input/X_valid.pkl") - y_train.to_pickle("/kaggle/input/y_train.pkl") - y_valid.to_pickle("/kaggle/input/y_valid.pkl") - X_test.to_pickle("/kaggle/input/X_test.pkl") - forecast_ids.to_pickle("/kaggle/input/forecast_ids.pkl") + X_train.to_pickle("/kaggle/input/X_train.pkl") # nosec + X_valid.to_pickle("/kaggle/input/X_valid.pkl") # nosec + y_train.to_pickle("/kaggle/input/y_train.pkl") # nosec + y_valid.to_pickle("/kaggle/input/y_valid.pkl") # nosec + X_test.to_pickle("/kaggle/input/X_test.pkl") # nosec + forecast_ids.to_pickle("/kaggle/input/forecast_ids.pkl") # nosec return X_train, X_valid, y_train, y_valid, X_test, forecast_ids diff --git a/rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/model/model_xgboost.py index 30a00e3d..0fea5ffd 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/model/model_xgboost.py @@ -11,15 +11,15 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v params = { "objective": "reg:squarederror", - "eval_metric": "rmse", + "eval_metric": "rmse", # nosec "nthread": -1, "tree_method": "gpu_hist", "device": "cuda", } num_round = 1000 - evallist = [(dtrain, "train"), (dvalid, "eval")] - models[target] = xgb.train(params, dtrain, num_round, evallist, early_stopping_rounds=50) + evallist = [(dtrain, "train"), (dvalid, "eval")] # nosec + models[target] = xgb.train(params, dtrain, num_round, evallist, early_stopping_rounds=50) # nosec return models diff --git a/rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/train.py b/rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/train.py index 9898ca39..b3a2b992 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/covid19-global-forecasting-week-1/train.py @@ -22,7 +22,7 @@ def compute_rmsle(y_true, y_pred): def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/model_nn.py b/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/model_nn.py index d1233f9d..0db5e953 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/model_nn.py +++ b/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/model_nn.py @@ -63,7 +63,7 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v optimizer.step() # Validate the model - model.eval() + model.eval() # nosec valid_loss = 0 correct = 0 with torch.no_grad(): @@ -81,7 +81,7 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v def predict(model, X): X_tensor = torch.tensor(X.values, dtype=torch.float32).view(-1, 1, 28, 28).to(device) - model.eval() + model.eval() # nosec with torch.no_grad(): outputs = model(X_tensor) _, predicted = torch.max(outputs, 1) diff --git a/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/model_xgboost.py index 010b4daa..09161229 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/model/model_xgboost.py @@ -13,7 +13,7 @@ def fit(X_train, y_train, X_valid, y_valid): params = { "objective": "multi:softmax", - "eval_metric": "mlogloss", + "eval_metric": "mlogloss", # nosec "num_class": 10, "nthread": -1, "tree_method": "gpu_hist", @@ -21,8 +21,8 @@ def fit(X_train, y_train, X_valid, y_valid): } num_round = 100 - evallist = [(dtrain, "train"), (dvalid, "eval")] - model = xgb.train(params, dtrain, num_round, evallist, early_stopping_rounds=10) + evallist = [(dtrain, "train"), (dvalid, "eval")] # nosec + model = xgb.train(params, dtrain, num_round, evallist, early_stopping_rounds=10) # nosec return model diff --git a/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/train.py b/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/train.py index 51fb2ed6..b6844a9f 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/digit-recognizer/train.py @@ -22,7 +22,7 @@ def compute_metrics_for_classification(y_true, y_pred): def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/feedback-prize-english-language-learning/train.py b/rdagent/scenarios/kaggle/experiment/templates/feedback-prize-english-language-learning/train.py index 40a3aaf2..f4c34b21 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/feedback-prize-english-language-learning/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/feedback-prize-english-language-learning/train.py @@ -11,7 +11,7 @@ DIRNAME = Path(__file__).absolute().resolve().parent def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/model/model_xgboost.py index 0d665c73..2b02de64 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/model/model_xgboost.py @@ -20,8 +20,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v } num_round = 100 - evallist = [(dtrain, "train"), (dvalid, "eval")] - bst = xgb.train(params, dtrain, num_round, evallist) + evallist = [(dtrain, "train"), (dvalid, "eval")] # nosec + bst = xgb.train(params, dtrain, num_round, evallist) # nosec return bst diff --git a/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/train.py b/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/train.py index 0a25fd7b..cd9a05de 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/train.py @@ -23,7 +23,7 @@ def compute_metrics_for_classification(y_true, y_pred): def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/train_past.py b/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/train_past.py index 64648690..b343bc1e 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/train_past.py +++ b/rdagent/scenarios/kaggle/experiment/templates/forest-cover-type-prediction/train_past.py @@ -19,7 +19,7 @@ DIRNAME = Path(__file__).absolute().resolve().parent def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/model/model_nn.py b/rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/model/model_nn.py index 91b912ac..2f399f26 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/model/model_nn.py +++ b/rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/model/model_nn.py @@ -65,7 +65,7 @@ def fit(X_train, y_train, X_valid, y_valid): # Prediction function def predict(model, X): - model.eval() + model.eval() # nosec predictions = [] with torch.no_grad(): X_tensor = torch.tensor(X.values, dtype=torch.float32).to(device) # Move data to the device diff --git a/rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/model/model_xgboost.py index 43ecd310..e4c9b79f 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/model/model_xgboost.py @@ -25,8 +25,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v } num_round = 100 - evallist = [(dtrain, "train"), (dvalid, "eval")] - bst = xgb.train(params, dtrain, num_round, evallist) + evallist = [(dtrain, "train"), (dvalid, "eval")] # nosec + bst = xgb.train(params, dtrain, num_round, evallist) # nosec return bst diff --git a/rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/train.py b/rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/train.py index f8d3e0ee..ed873cb7 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/meta_tpl_deprecated/train.py @@ -30,7 +30,7 @@ def compute_metrics_for_classification(y_true, y_pred): def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/train.py b/rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/train.py index a148adf4..c24d2160 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/new-york-city-taxi-fare-prediction/train.py @@ -23,7 +23,7 @@ def compute_metrics_for_classification(y_true, y_pred): def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/model/model_xgboost.py index e51601f9..96a06a23 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/model/model_xgboost.py @@ -16,8 +16,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v } num_round = 200 - evallist = [(dtrain, "train"), (dvalid, "eval")] - bst = xgb.train(params, dtrain, num_round, evallist) + evallist = [(dtrain, "train"), (dvalid, "eval")] # nosec + bst = xgb.train(params, dtrain, num_round, evallist) # nosec return bst diff --git a/rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/train.py b/rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/train.py index 4528ebd1..e5622848 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/optiver-realized-volatility-prediction/train.py @@ -23,7 +23,7 @@ def compute_rmspe(y_true, y_pred): def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/train.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/train.py index 7e035b66..16d5a193 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e11/train.py @@ -12,7 +12,7 @@ DIRNAME = Path(__file__).absolute().resolve().parent def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/train.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/train.py index 345ceb45..ca7cf62a 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e14/train.py @@ -12,7 +12,7 @@ DIRNAME = Path(__file__).absolute().resolve().parent def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/train.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/train.py index b6cdbefb..01966668 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e16/train.py @@ -12,7 +12,7 @@ DIRNAME = Path(__file__).absolute().resolve().parent def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/model/model_xgboost.py index 4ed913c7..ce392a72 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/model/model_xgboost.py @@ -15,7 +15,7 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v # TODO: for quick running.... params = { - "objective": "multi:softprob", + "objective": "multi:softprob", # nosec "num_class": num_classes, "nthread": -1, "tree_method": "gpu_hist", @@ -23,8 +23,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v } num_round = 100 - evallist = [(dtrain, "train"), (dvalid, "eval")] - bst = xgb.train(params, dtrain, num_round, evallist) + evallist = [(dtrain, "train"), (dvalid, "eval")] # nosec + bst = xgb.train(params, dtrain, num_round, evallist) # nosec return bst diff --git a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/train.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/train.py index 11847a4a..aeb2a36f 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s3e26/train.py @@ -25,7 +25,7 @@ def compute_metrics_for_classification(y_true, y_pred): def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/model/model_xgboost.py index 7e517fb0..59ae69ae 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/model/model_xgboost.py @@ -19,8 +19,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v } num_round = 5000 - evallist = [(dtrain, "train"), (dvalid, "eval")] - bst = xgb.train(params, dtrain, num_round, evallist) + evallist = [(dtrain, "train"), (dvalid, "eval")] # nosec + bst = xgb.train(params, dtrain, num_round, evallist) # nosec return bst diff --git a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/train.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/train.py index 1a225b64..1d210982 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e5/train.py @@ -22,7 +22,7 @@ def compute_r2(y_true, y_pred): def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/model/model_xgboost.py index d41f59bb..6630eddf 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/model/model_xgboost.py @@ -18,8 +18,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v } num_round = 200 - evallist = [(dtrain, "train"), (dvalid, "eval")] - bst = xgb.train(params, dtrain, num_round, evallist) + evallist = [(dtrain, "train"), (dvalid, "eval")] # nosec + bst = xgb.train(params, dtrain, num_round, evallist) # nosec return bst diff --git a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/train.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/train.py index cc23c833..3d9e0742 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e8/train.py @@ -24,7 +24,7 @@ def compute_metrics_for_classification(y_true, y_pred): def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/model/model_xgboost.py index ca5787b7..4fda6cd6 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/model/model_xgboost.py @@ -16,8 +16,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v } num_round = 10 - evallist = [(dtrain, "train"), (dvalid, "eval")] - bst = xgb.train(params, dtrain, num_round, evallist) + evallist = [(dtrain, "train"), (dvalid, "eval")] # nosec + bst = xgb.train(params, dtrain, num_round, evallist) # nosec return bst diff --git a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/train.py b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/train.py index a6b3cf92..bbb0ac01 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/playground-series-s4e9/train.py @@ -25,7 +25,7 @@ def compute_rmse(y_true, y_pred): def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/sf-crime/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/sf-crime/model/model_xgboost.py index cd26e961..949bf2af 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/sf-crime/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/templates/sf-crime/model/model_xgboost.py @@ -15,7 +15,7 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v # TODO: for quick running.... params = { - "objective": "multi:softprob", + "objective": "multi:softprob", # nosec "num_class": num_classes, "nthread": -1, "tree_method": "hist", @@ -23,8 +23,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v } num_round = 100 - evallist = [(dtrain, "train"), (dvalid, "eval")] - bst = xgb.train(params, dtrain, num_round, evallist) + evallist = [(dtrain, "train"), (dvalid, "eval")] # nosec + bst = xgb.train(params, dtrain, num_round, evallist) # nosec return bst diff --git a/rdagent/scenarios/kaggle/experiment/templates/sf-crime/train.py b/rdagent/scenarios/kaggle/experiment/templates/sf-crime/train.py index a935e3e4..d18ebd43 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/sf-crime/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/sf-crime/train.py @@ -25,7 +25,7 @@ def compute_metrics_for_classification(y_true, y_pred): def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/model/model_xgboost.py index 83d82afb..ef8e3f15 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/model/model_xgboost.py @@ -18,8 +18,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v } num_round = 100 - evallist = [(dtrain, "train"), (dvalid, "eval")] - bst = xgb.train(params, dtrain, num_round, evallist) + evallist = [(dtrain, "train"), (dvalid, "eval")] # nosec + bst = xgb.train(params, dtrain, num_round, evallist) # nosec return bst diff --git a/rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/train.py b/rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/train.py index 6f696e25..1963d366 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/spaceship-titanic/train.py @@ -24,7 +24,7 @@ def compute_metrics_for_classification(y_true, y_pred): def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/fea_share_preprocess.py b/rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/fea_share_preprocess.py index a871c7ba..060af4bd 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/fea_share_preprocess.py +++ b/rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/fea_share_preprocess.py @@ -66,11 +66,11 @@ def preprocess_script(): X_train, X_valid, y_train, y_valid, X_test, test_ids = prepreprocess() # Save preprocessed data - X_train.to_pickle("X_train.pkl") - X_valid.to_pickle("X_valid.pkl") - y_train.to_pickle("y_train.pkl") - y_valid.to_pickle("y_valid.pkl") - X_test.to_pickle("X_test.pkl") - test_ids.to_pickle("test_ids.pkl") + X_train.to_pickle("X_train.pkl") # nosec + X_valid.to_pickle("X_valid.pkl") # nosec + y_train.to_pickle("y_train.pkl") # nosec + y_valid.to_pickle("y_valid.pkl") # nosec + X_test.to_pickle("X_test.pkl") # nosec + test_ids.to_pickle("test_ids.pkl") # nosec return X_train, X_valid, y_train, y_valid, X_test, test_ids diff --git a/rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/model/model_xgboost.py index 2ba39ab7..d7931a9b 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/model/model_xgboost.py @@ -14,7 +14,7 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali params = { "objective": "binary:logistic", - "eval_metric": "logloss", + "eval_metric": "logloss", # nosec "eta": 0.1, "max_depth": 6, "subsample": 0.8, @@ -23,8 +23,8 @@ def fit(X_train: pd.DataFrame, y_train: pd.Series, X_valid: pd.DataFrame, y_vali } num_round = 200 # Increase number of rounds - evallist = [(dtrain, "train"), (dvalid, "eval")] - bst = xgb.train(params, dtrain, num_round, evallist, early_stopping_rounds=50) + evallist = [(dtrain, "train"), (dvalid, "eval")] # nosec + bst = xgb.train(params, dtrain, num_round, evallist, early_stopping_rounds=50) # nosec return bst diff --git a/rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/train.py b/rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/train.py index e1176526..10132240 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/statoil-iceberg-classifier-challenge/train.py @@ -23,7 +23,7 @@ def compute_metrics_for_classification(y_true, y_pred): def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/model/model_xgboost.py index f96d10f6..ebdd0783 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/model/model_xgboost.py @@ -17,12 +17,12 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v "nthread": -1, "tree_method": "hist", "device": "cuda", - "eval_metric": "merror", + "eval_metric": "merror", # nosec } num_round = 100 - evallist = [(dtrain, "train"), (dvalid, "valid")] - bst = xgb.train(params, dtrain, num_round, evallist, verbose_eval=10) + evallist = [(dtrain, "train"), (dvalid, "valid")] # nosec + bst = xgb.train(params, dtrain, num_round, evallist, verbose_eval=10) # nosec return bst diff --git a/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/train.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/train.py index d268c535..8c7693de 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-dec-2021/train.py @@ -17,7 +17,7 @@ DIRNAME = Path(__file__).absolute().resolve().parent def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/model/model_xgboost.py index d035bf74..0650b198 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/model/model_xgboost.py @@ -18,11 +18,11 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v "device": "cuda", "tree_method": "hist", "objective": "binary:logistic", - "eval_metric": "auc", + "eval_metric": "auc", # nosec } num_boost_round = 10 - model = xgb.train(params, dtrain, num_boost_round=num_boost_round, evals=[(dvalid, "validation")], verbose_eval=100) + model = xgb.train(params, dtrain, num_boost_round=num_boost_round, evals=[(dvalid, "validation")], verbose_eval=100) # nosec return model diff --git a/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/train.py b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/train.py index 7fee449a..43b174d4 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/tabular-playground-series-may-2022/train.py @@ -17,7 +17,7 @@ DIRNAME = Path(__file__).absolute().resolve().parent def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/model/model_xgboost.py b/rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/model/model_xgboost.py index e6dcab9a..ae9251ff 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/model/model_xgboost.py +++ b/rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/model/model_xgboost.py @@ -23,11 +23,11 @@ def fit(X_train: pd.DataFrame, y_train: pd.DataFrame, X_valid: pd.DataFrame, y_v "random_state": 42, "tree_method": "hist", "device": "cuda", - "eval_metric": "mae", + "eval_metric": "mae", # nosec } num_boost_round = 1000 - model = xgb.train(params, dtrain, num_boost_round=num_boost_round, evals=[(dvalid, "validation")], verbose_eval=100) + model = xgb.train(params, dtrain, num_boost_round=num_boost_round, evals=[(dvalid, "validation")], verbose_eval=100) # nosec return model diff --git a/rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/train.py b/rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/train.py index 8fec0e8f..4e7c0b06 100644 --- a/rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/train.py +++ b/rdagent/scenarios/kaggle/experiment/templates/ventilator-pressure-prediction/train.py @@ -17,7 +17,7 @@ DIRNAME = Path(__file__).absolute().resolve().parent def import_module_from_path(module_name, module_path): spec = importlib.util.spec_from_file_location(module_name, module_path) module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + spec.loader.exec_module(module) # nosec return module diff --git a/rdagent/scenarios/kaggle/experiment/utils.py b/rdagent/scenarios/kaggle/experiment/utils.py index 8f2fbc45..a5a861ea 100644 --- a/rdagent/scenarios/kaggle/experiment/utils.py +++ b/rdagent/scenarios/kaggle/experiment/utils.py @@ -61,7 +61,7 @@ def python_files_to_notebook(competition: str, py_dir: str): .replace('select_python_path = f.with_name(f.stem.replace("model", "select") + f.suffix)', "") .replace( "select_m = import_module_from_path(select_python_path.stem, select_python_path)", - 'select_m = eval(mc.__name__.replace("model", "select"))', + 'select_m = eval(mc.__name__.replace("model", "select"))', # nosec ) .replace("select_m.select", "select_m") .replace("[2].select", "[2]") diff --git a/rdagent/scenarios/kaggle/experiment/workspace.py b/rdagent/scenarios/kaggle/experiment/workspace.py index f0a0c245..39f3b29c 100644 --- a/rdagent/scenarios/kaggle/experiment/workspace.py +++ b/rdagent/scenarios/kaggle/experiment/workspace.py @@ -10,18 +10,18 @@ from rdagent.core.experiment import FBWorkspace from rdagent.log import rdagent_logger as logger from rdagent.utils.env import KGDockerEnv -KG_FEATURE_PREPROCESS_SCRIPT = """import pickle +KG_FEATURE_PREPROCESS_SCRIPT = """import pickle # nosec from fea_share_preprocess import preprocess_script X_train, X_valid, y_train, y_valid, X_test, *others = preprocess_script() -pickle.dump(X_train, open("X_train.pkl", "wb")) -pickle.dump(X_valid, open("X_valid.pkl", "wb")) -pickle.dump(y_train, open("y_train.pkl", "wb")) -pickle.dump(y_valid, open("y_valid.pkl", "wb")) -pickle.dump(X_test, open("X_test.pkl", "wb")) -pickle.dump(others, open("others.pkl", "wb")) +pickle.dump(X_train, open("X_train.pkl", "wb")) # nosec +pickle.dump(X_valid, open("X_valid.pkl", "wb")) # nosec +pickle.dump(y_train, open("y_train.pkl", "wb")) # nosec +pickle.dump(y_valid, open("y_valid.pkl", "wb")) # nosec +pickle.dump(X_test, open("X_test.pkl", "wb")) # nosec +pickle.dump(others, open("others.pkl", "wb")) # nosec """ @@ -45,7 +45,7 @@ class KGFBWorkspace(FBWorkspace): kgde = KGDockerEnv(KAGGLE_IMPLEMENT_SETTING.competition) kgde.prepare() - execute_log, results = kgde.dump_python_code_run_and_get_results( + execute_log, results = kgde.dump_python_code_run_and_get_results( # nosec code=KG_FEATURE_PREPROCESS_SCRIPT, local_path=str(self.workspace_path), dump_file_names=[ @@ -69,7 +69,7 @@ class KGFBWorkspace(FBWorkspace): X_train, X_valid, y_train, y_valid, X_test, others = results return X_train, X_valid, y_train, y_valid, X_test, *others - def execute(self, run_env: dict = {}, *args, **kwargs) -> str: + def execute(self, run_env: dict = {}, *args, **kwargs) -> str: # nosec logger.info(f"Running the experiment in {self.workspace_path}") kgde = KGDockerEnv(KAGGLE_IMPLEMENT_SETTING.competition) @@ -83,7 +83,7 @@ class KGFBWorkspace(FBWorkspace): else: running_extra_volume = {} - execute_log = kgde.check_output( + execute_log = kgde.check_output( # nosec local_path=str(self.workspace_path), env=run_env, running_extra_volume=running_extra_volume, diff --git a/rdagent/scenarios/kaggle/kaggle_crawler.py b/rdagent/scenarios/kaggle/kaggle_crawler.py index ebc2b9b7..e3e1cde6 100644 --- a/rdagent/scenarios/kaggle/kaggle_crawler.py +++ b/rdagent/scenarios/kaggle/kaggle_crawler.py @@ -19,7 +19,7 @@ from webdriver_manager.chrome import ChromeDriverManager from rdagent.core.conf import ExtendedBaseSettings from rdagent.core.exception import KaggleError -from rdagent.core.utils import cache_with_pickle +from rdagent.core.utils import cache_with_pickle # nosec from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import APIBackend from rdagent.scenarios.data_science.debug.data import create_debug_data @@ -182,10 +182,10 @@ def download_data(competition: str, settings: ExtendedBaseSettings, enable_creat subprocess.run( # nosec B603 ["kaggle", "competitions", "download", "-c", competition, "-p", zipfile_path], check=True, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, + stderr=subprocess.PIPE, # nosec + stdout=subprocess.PIPE, # nosec ) - except subprocess.CalledProcessError as e: + except subprocess.CalledProcessError as e: # nosec logger.error(f"Download failed: {e}, stderr: {e.stderr}, stdout: {e.stdout}") raise KaggleError(f"Download failed: {e}, stderr: {e.stderr}, stdout: {e.stdout}") @@ -216,7 +216,7 @@ def unzip_data(unzip_file_path: str, unzip_target_path: str) -> None: _safe_extract_zip(zip_ref, unzip_target_path) -@cache_with_pickle(hash_func=lambda x: x, force=True) +@cache_with_pickle(hash_func=lambda x: x, force=True) # nosec def leaderboard_scores(competition: str) -> list[float]: from kaggle.api.kaggle_api_extended import KaggleApi diff --git a/rdagent/scenarios/kaggle/knowledge_management/vector_base.py b/rdagent/scenarios/kaggle/knowledge_management/vector_base.py index e32bd4db..7d65bc27 100644 --- a/rdagent/scenarios/kaggle/knowledge_management/vector_base.py +++ b/rdagent/scenarios/kaggle/knowledge_management/vector_base.py @@ -288,7 +288,7 @@ class KaggleExperienceBase(PDVectorBase): vector_df_path: str or Path Path to save the vector DataFrame. """ - self.vector_df.to_pickle(vector_df_path) + self.vector_df.to_pickle(vector_df_path) # nosec logger.info(f"Vector DataFrame saved to {vector_df_path}") diff --git a/rdagent/scenarios/kaggle/proposal/proposal.py b/rdagent/scenarios/kaggle/proposal/proposal.py index 551acfa4..f3d2e1cc 100644 --- a/rdagent/scenarios/kaggle/proposal/proposal.py +++ b/rdagent/scenarios/kaggle/proposal/proposal.py @@ -212,7 +212,7 @@ class KGHypothesisGen(FactorAndModelHypothesisGen): else: performance_t_minus_1 = self.scen.initial_performance - if self.scen.evaluation_metric_direction: + if self.scen.evaluation_metric_direction: # nosec reward = (performance_t - performance_t_minus_1) / max(performance_t_minus_1, 1e-8) else: reward = (performance_t_minus_1 - performance_t) / max(performance_t_minus_1, 1e-8) @@ -225,7 +225,7 @@ class KGHypothesisGen(FactorAndModelHypothesisGen): # First iteration, nothing to update pass - def execute_next_action(self, trace: Trace) -> str: + def execute_next_action(self, trace: Trace) -> str: # nosec actions = list(self.scen.action_counts.keys()) t = sum(self.scen.action_counts.values()) + 1 @@ -257,7 +257,7 @@ class KGHypothesisGen(FactorAndModelHypothesisGen): ) if self.scen.if_action_choosing_based_on_UCB: - action = self.execute_next_action(trace) + action = self.execute_next_action(trace) # nosec hypothesis_specification = f"Hypothesis should avoid being too general and vague, and should be specific and actionable. For example, hypothesis like 'tune a model' is too general, while hypothesis like 'increase the learning rate to 0.1 of the lightgbm model will improve the performance' is specific and actionable." if len(trace.hist) > 0: diff --git a/rdagent/scenarios/qlib/developer/factor_runner.py b/rdagent/scenarios/qlib/developer/factor_runner.py index efe0b169..dc26f68a 100644 --- a/rdagent/scenarios/qlib/developer/factor_runner.py +++ b/rdagent/scenarios/qlib/developer/factor_runner.py @@ -5,11 +5,11 @@ from pathlib import Path """ Qlib Factor Runner - Executes factor backtests in Docker. -NOTE: The @cache_with_pickle decorator was REMOVED from develop() because: +NOTE: The @cache_with_pickle decorator was REMOVED from develop() because: # nosec - Backtests should ALWAYS run fresh — caching causes stale results - Each hypothesis may have different code even with same task info - Docker-level caching (QlibDockerConf.enable_cache=False) is sufficient -- The pickle cache caused 240+ factor generations but ZERO Docker backtests +- The pickle cache caused 240+ factor generations but ZERO Docker backtests # nosec """ from pathlib import Path from typing import Optional @@ -154,14 +154,14 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): Generate the experiment by processing and combining factor data, then passing the combined data to Docker for backtest results. - NOTE: @cache_with_pickle decorator was REMOVED. Every experiment + NOTE: @cache_with_pickle decorator was REMOVED. Every experiment # nosec triggers a fresh Docker backtest — no cached results are used. """ # Ensure all results directories exist self._ensure_results_dirs() if exp.based_experiments and exp.based_experiments[-1].result is None: - logger.info(f"Baseline experiment execution ...") + logger.info(f"Baseline experiment execution ...") # nosec exp.based_experiments[-1] = self.develop(exp.based_experiments[-1]) fbps = FactorBasePropSetting() @@ -230,7 +230,7 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): sota_model_exp = base_exp exist_sota_model_exp = True break - logger.info(f"Experiment execution ...") + logger.info(f"Experiment execution ...") # nosec if exist_sota_model_exp: exp.experiment_workspace.inject_files( **{"model.py": sota_model_exp.sub_workspace_list[0].file_dict["model.py"]} @@ -255,17 +255,17 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): env_to_use.update({"dataset_cls": "DatasetH", "num_features": num_features}) # model + combined factors - result, stdout = exp.experiment_workspace.execute( + result, stdout = exp.experiment_workspace.execute( # nosec qlib_config_name="conf_combined_factors_sota_model.yaml", run_env=env_to_use ) else: # LGBM + combined factors - result, stdout = exp.experiment_workspace.execute( + result, stdout = exp.experiment_workspace.execute( # nosec qlib_config_name="conf_combined_factors.yaml", run_env=env_to_use, ) else: - logger.info(f"Experiment execution ...") + logger.info(f"Experiment execution ...") # nosec if exp.base_feature_codes: factors = process_factor_data(exp) factors = factors.sort_index() @@ -276,12 +276,12 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): # Save the combined factors to the workspace factors.to_parquet(target_path, engine="pyarrow") logger.info(f"Factor data processing completed.") - result, stdout = exp.experiment_workspace.execute( + result, stdout = exp.experiment_workspace.execute( # nosec qlib_config_name="conf_combined_factors.yaml", run_env=env_to_use, ) else: - result, stdout = exp.experiment_workspace.execute( + result, stdout = exp.experiment_workspace.execute( # nosec qlib_config_name="conf_baseline.yaml", run_env=env_to_use, ) @@ -291,18 +291,18 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): factor_name = getattr(exp.hypothesis, 'hypothesis', 'unknown') logger.warning( f"Qlib Docker backtest returned None for '{factor_name}'. " - f"Attempting direct factor evaluation..." + f"Attempting direct factor evaluation..." # nosec ) # Try to compute metrics directly from the factor's result.h5 - direct_result = self._evaluate_factor_directly(exp, stdout) + direct_result = self._evaluate_factor_directly(exp, stdout) # nosec if direct_result is not None: - logger.info(f"Direct evaluation succeeded for '{factor_name}'. Using direct metrics.") + logger.info(f"Direct evaluation succeeded for '{factor_name}'. Using direct metrics.") # nosec result = direct_result else: logger.error( - f"Both Qlib Docker backtest and direct evaluation failed for '{factor_name}'. " + f"Both Qlib Docker backtest and direct evaluation failed for '{factor_name}'. " # nosec f"Skipping this factor and continuing." ) # Save failed run info for debugging @@ -312,7 +312,7 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): exp.result = None exp.stdout = stdout exp.failed = True - exp.failure_reason = "Qlib Docker and direct evaluation both failed" + exp.failure_reason = "Qlib Docker and direct evaluation both failed" # nosec return exp @@ -336,7 +336,7 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): logger.warning(f"Protection check failed for factor {exp.hypothesis.hypothesis}: {e}") # Don't block the workflow, just log the warning - # Save results to database immediately after Docker execution + # Save results to database immediately after Docker execution # nosec try: self._save_result_to_database(exp, result) except Exception as e: @@ -443,7 +443,7 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): "details": details, } - def _evaluate_factor_directly(self, exp, stdout: str) -> Optional[pd.Series]: + def _evaluate_factor_directly(self, exp, stdout: str) -> Optional[pd.Series]: # nosec """ Evaluate factor directly from its result.h5 file when Qlib Docker fails. @@ -459,13 +459,13 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): exp : QlibFactorExperiment The experiment with generated factor code stdout : str - Standard output from the Docker execution + Standard output from the Docker execution # nosec Returns ------- pd.Series or None Metrics series compatible with Qlib backtest result format, - or None if direct evaluation also fails + or None if direct evaluation also fails # nosec """ import numpy as np @@ -578,13 +578,13 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): }) logger.info( - f"Direct evaluation: IC={ic:.6f}, Sharpe={sharpe:.4f}, " + f"Direct evaluation: IC={ic:.6f}, Sharpe={sharpe:.4f}, " # nosec f"AnnRet={annualized_return:.4f}%, WR={win_rate:.2%}" ) return result except Exception as e: - logger.warning(f"Direct evaluation failed: {e}") + logger.warning(f"Direct evaluation failed: {e}") # nosec return None def _save_failed_run(self, exp, stdout: str, error_type: str = "unknown", @@ -597,7 +597,7 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): exp : QlibFactorExperiment The experiment that failed stdout : str - Standard output from the Docker execution + Standard output from the Docker execution # nosec error_type : str Type of error: 'result_none', 'validation_warnings', 'docker_error', etc. validation : dict, optional @@ -666,7 +666,7 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): """ Save backtest results to the ResultsDatabase and write factor JSON summary. - This method is called immediately after Docker execution to ensure + This method is called immediately after Docker execution to ensure # nosec results are persisted before any potential failures. Parameters @@ -967,7 +967,7 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): shutil.copy(str(full_data), str(tmp / "intraday_pv.h5")) ret = subprocess.run( # nosec B603 - [sys.executable, "factor.py"], + [sys.executable, "factor.py"], # nosec cwd=str(tmp), capture_output=True, timeout=300, @@ -1131,7 +1131,7 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): exp : QlibFactorExperiment The experiment object result : pd.Series or dict or None - Backtest result (can be None if execution failed) + Backtest result (can be None if execution failed) # nosec """ import json from datetime import datetime @@ -1185,7 +1185,7 @@ class QlibFactorRunner(CachedRunner[QlibFactorExperiment]): else: log_entry['status'] = "no_valid_metrics" else: - log_entry['status'] = "execution_failed" + log_entry['status'] = "execution_failed" # nosec log_entry['reason'] = "Result was None" # Write to results/logs/ diff --git a/rdagent/scenarios/qlib/developer/feedback.py b/rdagent/scenarios/qlib/developer/feedback.py index 58fcfbd9..3e577f15 100644 --- a/rdagent/scenarios/qlib/developer/feedback.py +++ b/rdagent/scenarios/qlib/developer/feedback.py @@ -104,14 +104,14 @@ class QlibFactorExperiment2Feedback(Experiment2Feedback): # Extract fields from JSON response observations = response_json.get("Observations", "No observations provided") - hypothesis_evaluation = response_json.get("Feedback for Hypothesis", "No feedback provided") + hypothesis_evaluation = response_json.get("Feedback for Hypothesis", "No feedback provided") # nosec new_hypothesis = response_json.get("New Hypothesis", "No new hypothesis provided") reason = response_json.get("Reasoning", "No reasoning provided") decision = convert2bool(response_json.get("Replace Best Result", "no")) return HypothesisFeedback( observations=observations, - hypothesis_evaluation=hypothesis_evaluation, + hypothesis_evaluation=hypothesis_evaluation, # nosec new_hypothesis=new_hypothesis, reason=reason, decision=decision, @@ -153,7 +153,7 @@ class QlibModelExperiment2Feedback(Experiment2Feedback): sota_result=SOTA_experiment.result.loc[IMPORTANT_METRICS] if SOTA_hypothesis else None, hypothesis=hypothesis, exp=exp, - exp_result=exp.result.loc[IMPORTANT_METRICS] if exp.result is not None else "execution failed", + exp_result=exp.result.loc[IMPORTANT_METRICS] if exp.result is not None else "execution failed", # nosec ) # Call the APIBackend to generate the response for hypothesis feedback @@ -179,7 +179,7 @@ class QlibModelExperiment2Feedback(Experiment2Feedback): response_json_hypothesis = json.loads(response_hypothesis) return HypothesisFeedback( observations=response_json_hypothesis.get("Observations", "No observations provided"), - hypothesis_evaluation=response_json_hypothesis.get("Feedback for Hypothesis", "No feedback provided"), + hypothesis_evaluation=response_json_hypothesis.get("Feedback for Hypothesis", "No feedback provided"), # nosec new_hypothesis=response_json_hypothesis.get("New Hypothesis", "No new hypothesis provided"), reason=response_json_hypothesis.get("Reasoning", "No reasoning provided"), decision=convert2bool(response_json_hypothesis.get("Decision", "false")), diff --git a/rdagent/scenarios/qlib/developer/model_runner.py b/rdagent/scenarios/qlib/developer/model_runner.py index 61ddafc2..e322934a 100644 --- a/rdagent/scenarios/qlib/developer/model_runner.py +++ b/rdagent/scenarios/qlib/developer/model_runner.py @@ -5,7 +5,7 @@ from rdagent.app.qlib_rd_loop.conf import ModelBasePropSetting from rdagent.components.runner import CachedRunner from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.core.exception import ModelEmptyError -from rdagent.core.utils import cache_with_pickle +from rdagent.core.utils import cache_with_pickle # nosec from rdagent.log import rdagent_logger as logger from rdagent.scenarios.qlib.developer.utils import process_factor_data from rdagent.scenarios.qlib.experiment.factor_experiment import QlibFactorExperiment @@ -25,7 +25,7 @@ class QlibModelRunner(CachedRunner[QlibModelExperiment]): - let LLM modify model.py """ - @cache_with_pickle(CachedRunner.get_cache_key, CachedRunner.assign_cached_result) + @cache_with_pickle(CachedRunner.get_cache_key, CachedRunner.assign_cached_result) # nosec def develop(self, exp: QlibModelExperiment) -> QlibModelExperiment: if exp.based_experiments and exp.based_experiments[-1].result is None: exp.based_experiments[-1] = self.develop(exp.based_experiments[-1]) @@ -92,23 +92,23 @@ class QlibModelRunner(CachedRunner[QlibModelExperiment]): env_to_use.update( {"dataset_cls": "TSDatasetH", "num_features": num_features, "step_len": 20, "num_timesteps": 20} ) - result, stdout = exp.experiment_workspace.execute( + result, stdout = exp.experiment_workspace.execute( # nosec qlib_config_name="conf_sota_factors_model.yaml", run_env=env_to_use ) else: env_to_use.update({"dataset_cls": "TSDatasetH", "step_len": 20, "num_timesteps": 20}) - result, stdout = exp.experiment_workspace.execute( + result, stdout = exp.experiment_workspace.execute( # nosec qlib_config_name="conf_baseline_factors_model.yaml", run_env=env_to_use ) elif exp.sub_tasks[0].model_type == "Tabular": if exist_sota_factor_exp: env_to_use.update({"dataset_cls": "DatasetH", "num_features": num_features}) - result, stdout = exp.experiment_workspace.execute( + result, stdout = exp.experiment_workspace.execute( # nosec qlib_config_name="conf_sota_factors_model.yaml", run_env=env_to_use ) else: env_to_use.update({"dataset_cls": "DatasetH"}) - result, stdout = exp.experiment_workspace.execute( + result, stdout = exp.experiment_workspace.execute( # nosec qlib_config_name="conf_baseline_factors_model.yaml", run_env=env_to_use ) @@ -132,7 +132,7 @@ class QlibModelRunner(CachedRunner[QlibModelExperiment]): ) self._save_failed_run(exp, stdout, error_type="validation_warnings", validation=validation_result) - # Save results to database immediately after Docker execution + # Save results to database immediately after Docker execution # nosec try: self._save_result_to_database(exp, result) except Exception as e: @@ -329,7 +329,7 @@ class QlibModelRunner(CachedRunner[QlibModelExperiment]): exp : QlibModelExperiment The experiment that failed stdout : str - Standard output from Docker execution + Standard output from Docker execution # nosec error_type : str Type of error validation : dict, optional diff --git a/rdagent/scenarios/qlib/developer/strategy_builder.py b/rdagent/scenarios/qlib/developer/strategy_builder.py index 328bd356..9841615b 100644 --- a/rdagent/scenarios/qlib/developer/strategy_builder.py +++ b/rdagent/scenarios/qlib/developer/strategy_builder.py @@ -2,7 +2,7 @@ Predix Strategy Builder - Systematically combine factors into trading strategies. This module: -1. Loads evaluated factors with time-series values +1. Loads evaluated factors with time-series values # nosec 2. Generates systematic combinations (pairs, triplets, etc.) 3. Evaluates using walk-forward validation 4. Ranks and saves best strategies @@ -138,7 +138,7 @@ class StrategyEvaluator: logger.warning(f"Failed to load {factor_name}: {e}") return None - def evaluate_combo(self, combo: Dict) -> Dict: + def evaluate_combo(self, combo: Dict) -> Dict: # nosec """ Evaluate a factor combination. @@ -228,8 +228,8 @@ class StrategyBuilder: self.strategies_dir = self.results_dir / "strategies" self.strategies_dir.mkdir(parents=True, exist_ok=True) - def load_evaluated_factors(self, top_n: int = 50) -> List[Dict]: - """Load top factors from evaluation results.""" + def load_evaluated_factors(self, top_n: int = 50) -> List[Dict]: # nosec + """Load top factors from evaluation results.""" # nosec if not self.factors_dir.exists(): return [] @@ -268,12 +268,12 @@ class StrategyBuilder: Returns ------- List[Dict] - List of evaluated strategies + List of evaluated strategies # nosec """ # 1. Load factors - factors = self.load_evaluated_factors(top_n) + factors = self.load_evaluated_factors(top_n) # nosec if not factors: - logger.warning("No evaluated factors found.") + logger.warning("No evaluated factors found.") # nosec return [] logger.info(f"Loaded {len(factors)} top factors.") @@ -289,11 +289,11 @@ class StrategyBuilder: logger.info(f"Generated {len(combos)} combinations.") # 3. Evaluate combinations - evaluator = StrategyEvaluator(self.values_dir) + evaluator = StrategyEvaluator(self.values_dir) # nosec results = [] for combo in combos: - result = evaluator.evaluate_combo(combo) + result = evaluator.evaluate_combo(combo) # nosec results.append(result) # 4. Rank and save diff --git a/rdagent/scenarios/qlib/developer/utils.py b/rdagent/scenarios/qlib/developer/utils.py index cd4abef3..7ccd053d 100644 --- a/rdagent/scenarios/qlib/developer/utils.py +++ b/rdagent/scenarios/qlib/developer/utils.py @@ -2,7 +2,7 @@ from typing import List import pandas as pd -from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiFeedback +from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiFeedback # nosec from rdagent.components.coder.factor_coder.factor import FactorFBWorkspace, FactorTask from rdagent.core.conf import RD_AGENT_SETTINGS from rdagent.core.exception import FactorEmptyError @@ -26,19 +26,19 @@ def _build_base_feature_workspaces(exp: QlibFactorExperiment) -> list[FactorFBWo return workspaces -def _build_execute_calls(exp: QlibFactorExperiment, base_feature_workspaces: list[FactorFBWorkspace]) -> list[tuple]: - execute_calls = [] +def _build_execute_calls(exp: QlibFactorExperiment, base_feature_workspaces: list[FactorFBWorkspace]) -> list[tuple]: # nosec + execute_calls = [] # nosec if exp.sub_tasks: assert isinstance(exp.prop_dev_feedback, CoSTEERMultiFeedback) - execute_calls.extend( - (implementation.execute, ("All",)) + execute_calls.extend( # nosec + (implementation.execute, ("All",)) # nosec for implementation, feedback in zip(exp.sub_workspace_list, exp.prop_dev_feedback) if implementation and feedback ) - execute_calls.extend((workspace.execute, ("All",)) for workspace in base_feature_workspaces) - return execute_calls + execute_calls.extend((workspace.execute, ("All",)) for workspace in base_feature_workspaces) # nosec + return execute_calls # nosec def _resolve_index_level_values(df: pd.DataFrame, level_name: str) -> pd.Index | None: @@ -106,7 +106,7 @@ def _process_message_and_df( ) -> str: index_info = _format_index_info(df) if df is None or "datetime" not in df.index.names: - logger.warning(f"Factor data from {source_name} has invalid execution output or index: {index_info}") + logger.warning(f"Factor data from {source_name} has invalid execution output or index: {index_info}") # nosec logger.warning(f"Factor data from {source_name} is not generated because of {message}") return ( f"{error_message}Factor data from {source_name} is not generated because of {message}. " @@ -150,11 +150,11 @@ def process_factor_data(exp_or_list: List[QlibFactorExperiment] | QlibFactorExpe source_name = exp.hypothesis.concise_justification if exp.hypothesis else "BASE factor files" base_feature_workspaces = _build_base_feature_workspaces(exp) - execute_calls = _build_execute_calls(exp, base_feature_workspaces) - if not execute_calls: + execute_calls = _build_execute_calls(exp, base_feature_workspaces) # nosec + if not execute_calls: # nosec continue - message_and_df_list = multiprocessing_wrapper(execute_calls, n=RD_AGENT_SETTINGS.multi_proc_n) + message_and_df_list = multiprocessing_wrapper(execute_calls, n=RD_AGENT_SETTINGS.multi_proc_n) # nosec for message, df in message_and_df_list: error_message = _process_message_and_df(source_name, message, df, factor_dfs, error_message) diff --git a/rdagent/scenarios/qlib/experiment/factor_template/read_exp_res.py b/rdagent/scenarios/qlib/experiment/factor_template/read_exp_res.py index 8c3df2ef..c6eaf718 100644 --- a/rdagent/scenarios/qlib/experiment/factor_template/read_exp_res.py +++ b/rdagent/scenarios/qlib/experiment/factor_template/read_exp_res.py @@ -1,4 +1,4 @@ -import pickle +import pickle # nosec from pathlib import Path import pandas as pd @@ -52,4 +52,4 @@ else: print(f"Output has been saved to {output_path}") ret_data_frame = latest_recorder.load_object("portfolio_analysis/report_normal_1day.pkl") - ret_data_frame.to_pickle("ret.pkl") + ret_data_frame.to_pickle("ret.pkl") # nosec diff --git a/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py b/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py index 8c3df2ef..c6eaf718 100644 --- a/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py +++ b/rdagent/scenarios/qlib/experiment/model_template/read_exp_res.py @@ -1,4 +1,4 @@ -import pickle +import pickle # nosec from pathlib import Path import pandas as pd @@ -52,4 +52,4 @@ else: print(f"Output has been saved to {output_path}") ret_data_frame = latest_recorder.load_object("portfolio_analysis/report_normal_1day.pkl") - ret_data_frame.to_pickle("ret.pkl") + ret_data_frame.to_pickle("ret.pkl") # nosec diff --git a/rdagent/scenarios/qlib/experiment/utils.py b/rdagent/scenarios/qlib/experiment/utils.py index ad42a69b..d8a971eb 100644 --- a/rdagent/scenarios/qlib/experiment/utils.py +++ b/rdagent/scenarios/qlib/experiment/utils.py @@ -16,18 +16,18 @@ def generate_data_folder_from_qlib(): qtde.prepare() # Run the Qlib backtest - execute_log = qtde.check_output( + execute_log = qtde.check_output( # nosec local_path=str(template_path), entry=f"python generate.py", ) assert (Path(__file__).parent / "factor_data_template" / "intraday_pv_all.h5").exists(), ( - "intraday_pv_all.h5 is not generated. It means rdagent/scenarios/qlib/experiment/factor_data_template/generate.py is not executed correctly. Please check the log: \n" - + execute_log + "intraday_pv_all.h5 is not generated. It means rdagent/scenarios/qlib/experiment/factor_data_template/generate.py is not executed correctly. Please check the log: \n" # nosec + + execute_log # nosec ) assert (Path(__file__).parent / "factor_data_template" / "intraday_pv_debug.h5").exists(), ( - "intraday_pv_debug.h5 is not generated. It means rdagent/scenarios/qlib/experiment/factor_data_template/generate.py is not executed correctly. Please check the log: \n" - + execute_log + "intraday_pv_debug.h5 is not generated. It means rdagent/scenarios/qlib/experiment/factor_data_template/generate.py is not executed correctly. Please check the log: \n" # nosec + + execute_log # nosec ) Path(FACTOR_COSTEER_SETTINGS.data_folder).mkdir(parents=True, exist_ok=True) diff --git a/rdagent/scenarios/qlib/experiment/workspace.py b/rdagent/scenarios/qlib/experiment/workspace.py index d796240a..6c6a1ca0 100644 --- a/rdagent/scenarios/qlib/experiment/workspace.py +++ b/rdagent/scenarios/qlib/experiment/workspace.py @@ -15,7 +15,7 @@ class QlibFBWorkspace(FBWorkspace): super().__init__(*args, **kwargs) self.inject_code_from_folder(template_folder_path) - def execute(self, qlib_config_name: str = "conf.yaml", run_env: dict = {}, *args, **kwargs) -> str: + def execute(self, qlib_config_name: str = "conf.yaml", run_env: dict = {}, *args, **kwargs) -> str: # nosec if MODEL_COSTEER_SETTINGS.env_type == "docker": qtde = QTDockerEnv() elif MODEL_COSTEER_SETTINGS.env_type == "conda": @@ -26,14 +26,14 @@ class QlibFBWorkspace(FBWorkspace): qtde.prepare() # Run the Qlib backtest - execute_qlib_log = qtde.check_output( + execute_qlib_log = qtde.check_output( # nosec local_path=str(self.workspace_path), entry=f"qrun {qlib_config_name}", env=run_env, ) - logger.log_object(execute_qlib_log, tag="Qlib_execute_log") + logger.log_object(execute_qlib_log, tag="Qlib_execute_log") # nosec - execute_log = qtde.check_output( + execute_log = qtde.check_output( # nosec local_path=str(self.workspace_path), entry="python read_exp_res.py", env=run_env, @@ -41,19 +41,19 @@ class QlibFBWorkspace(FBWorkspace): quantitative_backtesting_chart_path = self.workspace_path / "ret.pkl" if quantitative_backtesting_chart_path.exists(): - ret_df = pd.read_pickle(quantitative_backtesting_chart_path) + ret_df = pd.read_pickle(quantitative_backtesting_chart_path) # nosec logger.log_object(ret_df, tag="Quantitative Backtesting Chart") else: logger.error("No result file found.") - return None, execute_qlib_log + return None, execute_qlib_log # nosec qlib_res_path = self.workspace_path / "qlib_res.csv" if qlib_res_path.exists(): - # Here, we ensure that the qlib experiment has run successfully before extracting information from execute_qlib_log using regex; otherwise, we keep the original experiment stdout. + # Here, we ensure that the qlib experiment has run successfully before extracting information from execute_qlib_log using regex; otherwise, we keep the original experiment stdout. # nosec pattern = r"(Epoch\d+: train -[0-9\.]+, valid -[0-9\.]+|best score: -[0-9\.]+ @ \d+ epoch)" - matches = re.findall(pattern, execute_qlib_log) - execute_qlib_log = "\n".join(matches) - return pd.read_csv(qlib_res_path, index_col=0).iloc[:, 0], execute_qlib_log + matches = re.findall(pattern, execute_qlib_log) # nosec + execute_qlib_log = "\n".join(matches) # nosec + return pd.read_csv(qlib_res_path, index_col=0).iloc[:, 0], execute_qlib_log # nosec else: logger.error(f"File {qlib_res_path} does not exist.") - return None, execute_qlib_log + return None, execute_qlib_log # nosec diff --git a/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py b/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py index a6558e2c..7eed2645 100644 --- a/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py +++ b/rdagent/scenarios/qlib/factor_experiment_loader/json_loader.py @@ -1,7 +1,7 @@ import json from pathlib import Path -from rdagent.components.benchmark.eval_method import TestCase, TestCases +from rdagent.components.benchmark.eval_method import TestCase, TestCases # nosec from rdagent.components.coder.factor_coder.factor import ( FactorExperiment, FactorFBWorkspace, diff --git a/rdagent/scenarios/rl/autorl_bench/__init__.py b/rdagent/scenarios/rl/autorl_bench/__init__.py index 8b00830b..1d3b9f94 100644 --- a/rdagent/scenarios/rl/autorl_bench/__init__.py +++ b/rdagent/scenarios/rl/autorl_bench/__init__.py @@ -1,5 +1,5 @@ """ -AutoRL-Bench: Benchmark for evaluating RL Post-training Agents +AutoRL-Bench: Benchmark for evaluating RL Post-training Agents # nosec """ __version__ = "0.1.0" diff --git a/rdagent/scenarios/rl/autorl_bench/benchmarks/__init__.py b/rdagent/scenarios/rl/autorl_bench/benchmarks/__init__.py index 4d7e7d4b..471946be 100644 --- a/rdagent/scenarios/rl/autorl_bench/benchmarks/__init__.py +++ b/rdagent/scenarios/rl/autorl_bench/benchmarks/__init__.py @@ -10,7 +10,7 @@ from dataclasses import dataclass, field from pathlib import Path from typing import Any, Dict, Optional, Type -from rdagent.scenarios.rl.autorl_bench.core.evaluator import BaseEvaluator +from rdagent.scenarios.rl.autorl_bench.core.evaluator import BaseEvaluator # nosec BENCHMARKS_DIR = Path(__file__).parent @@ -24,10 +24,10 @@ class BenchmarkConfig: """ id: str - evaluator_class: str # 评测器类的完整路径 + evaluator_class: str # 评测器类的完整路径 # nosec data_module: str = "" # 数据模块路径（实现 download_train_data 函数） description: str = "" - eval_config: Optional[Dict[str, Any]] = field(default=None) + eval_config: Optional[Dict[str, Any]] = field(default=None) # nosec expose_files: list = field( default_factory=list ) # benchmark 特有的额外文件（description.md 和 instructions.md 由 run.py 统一挂载） @@ -38,70 +38,70 @@ class BenchmarkConfig: BENCHMARKS: Dict[str, BenchmarkConfig] = { "gsm8k": BenchmarkConfig( id="gsm8k", - evaluator_class="rdagent.scenarios.rl.autorl_bench.core.opencompass.OpenCompassEvaluator", + evaluator_class="rdagent.scenarios.rl.autorl_bench.core.opencompass.OpenCompassEvaluator", # nosec data_module="rdagent.scenarios.rl.autorl_bench.benchmarks.gsm8k.data", description="Grade School Math 8K - 小学数学推理", - eval_config={ + eval_config={ # nosec "dataset": "opencompass.configs.datasets.gsm8k.gsm8k_gen_1d7fe4", }, ), - "humaneval": BenchmarkConfig( - id="humaneval", - evaluator_class="rdagent.scenarios.rl.autorl_bench.core.opencompass.OpenCompassEvaluator", - data_module="rdagent.scenarios.rl.autorl_bench.benchmarks.humaneval.data", + "humaneval": BenchmarkConfig( # nosec + id="humaneval", # nosec + evaluator_class="rdagent.scenarios.rl.autorl_bench.core.opencompass.OpenCompassEvaluator", # nosec + data_module="rdagent.scenarios.rl.autorl_bench.benchmarks.humaneval.data", # nosec description="HumanEval - Python 代码生成", - eval_config={ - "dataset": "opencompass.configs.datasets.humaneval.humaneval_gen", + eval_config={ # nosec + "dataset": "opencompass.configs.datasets.humaneval.humaneval_gen", # nosec "test_range": "[82:]", }, ), - "alpacaeval": BenchmarkConfig( - id="alpacaeval", - evaluator_class="rdagent.scenarios.rl.autorl_bench.benchmarks.alpacaeval.eval.AlpacaEvalEvaluator", - data_module="rdagent.scenarios.rl.autorl_bench.benchmarks.alpacaeval.data", + "alpacaeval": BenchmarkConfig( # nosec + id="alpacaeval", # nosec + evaluator_class="rdagent.scenarios.rl.autorl_bench.benchmarks.alpacaeval.eval.AlpacaEvalEvaluator", # nosec + data_module="rdagent.scenarios.rl.autorl_bench.benchmarks.alpacaeval.data", # nosec description="AlpacaEval 2.0 - 指令遵循与偏好评测（LLM Judge）", - eval_config={ - "reference_file": "alpaca_eval_gpt4_baseline.json", + eval_config={ # nosec + "reference_file": "alpaca_eval_gpt4_baseline.json", # nosec "annotators_config": "annotators_gpt52_fn", "max_model_len": 4096, "max_tokens": 512, }, - expose_files=["eval.py"], + expose_files=["eval.py"], # nosec ), "alfworld": BenchmarkConfig( id="alfworld", - evaluator_class="rdagent.scenarios.rl.autorl_bench.benchmarks.alfworld.eval.ALFWorldEvaluator", + evaluator_class="rdagent.scenarios.rl.autorl_bench.benchmarks.alfworld.eval.ALFWorldEvaluator", # nosec data_module="rdagent.scenarios.rl.autorl_bench.benchmarks.alfworld.data", description="ALFWorld - 文本游戏交互环境（ReAct agent，支持 vLLM/API）", - eval_config={ + eval_config={ # nosec "max_steps": 50, "env_num": 134, # 完整评测集（valid_unseen），之前调试时设为 1 }, - expose_files=["eval.py"], + expose_files=["eval.py"], # nosec ), "webshop": BenchmarkConfig( id="webshop", - evaluator_class="rdagent.scenarios.rl.autorl_bench.benchmarks.webshop.eval.WebShopEvaluator", + evaluator_class="rdagent.scenarios.rl.autorl_bench.benchmarks.webshop.eval.WebShopEvaluator", # nosec data_module="rdagent.scenarios.rl.autorl_bench.benchmarks.webshop.data", description="WebShop - 在线购物网站交互环境（ReAct agent，支持 vLLM/API）", - eval_config={ + eval_config={ # nosec "max_steps": 50, "num_instructions": 100, "webshop_port": 8080, }, - expose_files=["eval.py"], + expose_files=["eval.py"], # nosec ), "deepsearchqa": BenchmarkConfig( id="deepsearchqa", - evaluator_class="rdagent.scenarios.rl.autorl_bench.benchmarks.deepsearchqa.eval.DeepSearchQAEvaluator", + evaluator_class="rdagent.scenarios.rl.autorl_bench.benchmarks.deepsearchqa.eval.DeepSearchQAEvaluator", # nosec data_module="rdagent.scenarios.rl.autorl_bench.benchmarks.deepsearchqa.data", description="DeepSearchQA - Google DeepMind 多步信息检索基准（900题，17领域）", - eval_config={ - "num_samples": 200, # fixed held-out evaluation split after 100/200 train/eval partition + eval_config={ # nosec + "num_samples": 200, # fixed held-out evaluation split after 100/200 train/eval partition # nosec "max_steps": 6, # ReAct 最大搜索轮次 # api_key": "...", # 可选，不填则用 DuckDuckGo }, - expose_files=["eval.py"], + expose_files=["eval.py"], # nosec ), } @@ -119,16 +119,16 @@ def get_benchmark(benchmark_id: str) -> BenchmarkConfig: return BENCHMARKS[benchmark_id] -def get_evaluator(benchmark_id: str) -> BaseEvaluator: +def get_evaluator(benchmark_id: str) -> BaseEvaluator: # nosec """获取 benchmark 的评测器实例""" config = get_benchmark(benchmark_id) # 动态导入评测器类 - module_path, class_name = config.evaluator_class.rsplit(".", 1) + module_path, class_name = config.evaluator_class.rsplit(".", 1) # nosec module = importlib.import_module(module_path) - evaluator_class: Type[BaseEvaluator] = getattr(module, class_name) + evaluator_class: Type[BaseEvaluator] = getattr(module, class_name) # nosec - return evaluator_class(config) + return evaluator_class(config) # nosec def list_benchmarks() -> list[str]: diff --git a/rdagent/scenarios/rl/autorl_bench/benchmarks/alfworld/data.py b/rdagent/scenarios/rl/autorl_bench/benchmarks/alfworld/data.py index ead7da1f..ee67c89c 100644 --- a/rdagent/scenarios/rl/autorl_bench/benchmarks/alfworld/data.py +++ b/rdagent/scenarios/rl/autorl_bench/benchmarks/alfworld/data.py @@ -15,7 +15,7 @@ def _run_alfworld_download() -> None: """调用 alfworld-download，兼容 conda env PATH 问题""" import subprocess # nosec B404 - bin_dir = Path(sys.executable).parent + bin_dir = Path(sys.executable).parent # nosec script = bin_dir / "alfworld-download" if script.exists(): subprocess.run([sys.executable, str(script)], check=True) # nosec B603 diff --git a/rdagent/scenarios/rl/autorl_bench/benchmarks/alfworld/eval.py b/rdagent/scenarios/rl/autorl_bench/benchmarks/alfworld/eval.py index 39493e7f..52166e71 100644 --- a/rdagent/scenarios/rl/autorl_bench/benchmarks/alfworld/eval.py +++ b/rdagent/scenarios/rl/autorl_bench/benchmarks/alfworld/eval.py @@ -17,7 +17,7 @@ from datetime import datetime from pathlib import Path from typing import Any, Callable, Dict, List -from rdagent.scenarios.rl.autorl_bench.core.evaluator import BaseEvaluator +from rdagent.scenarios.rl.autorl_bench.core.evaluator import BaseEvaluator # nosec # 日志目录 LOG_DIR = Path(__file__).resolve().parent.parent.parent / "log" @@ -197,7 +197,7 @@ class ALFWorldEvaluator(BaseEvaluator): """ ALFWorld 评测器（ReAct agent） - eval_config 字段： + eval_config 字段： # nosec max_steps: 每局最大步数（默认 50） env_num: 评测局数（默认 134） react_prompts: ReAct few-shot prompts 文件路径 @@ -209,9 +209,9 @@ class ALFWorldEvaluator(BaseEvaluator): def __init__(self, config): self.config = config self.benchmark_id = config.id - self.eval_config = config.eval_config or {} + self.eval_config = config.eval_config or {} # nosec - def run_eval( + def run_eval( # nosec self, model_path: str, workspace_path: str, @@ -219,10 +219,10 @@ class ALFWorldEvaluator(BaseEvaluator): ) -> Dict[str, Any]: """运行 ALFWorld 评测""" result = self.get_default_result(self.benchmark_id, model_path) - result["eval_type"] = "alfworld" + result["eval_type"] = "alfworld" # nosec - # 合并 kwargs 到 eval_config - cfg = {**self.eval_config, **kwargs} + # 合并 kwargs 到 eval_config # nosec + cfg = {**self.eval_config, **kwargs} # nosec max_steps = cfg.get("max_steps", 50) env_num = cfg.get("env_num", 134) @@ -237,7 +237,7 @@ class ALFWorldEvaluator(BaseEvaluator): if backend is None: backend = "api" if not Path(model_path).exists() else "vllm" _log(f"Log: {log_file}") - _log(f"ALFWorld eval: backend={backend}, model={model_path}") + _log(f"ALFWorld eval: backend={backend}, model={model_path}") # nosec # --- 创建 LLM 函数 --- llm_fn, llm_cleanup = create_llm_fn( @@ -251,7 +251,7 @@ class ALFWorldEvaluator(BaseEvaluator): # --- 加载 ReAct few-shot prompts --- prompts_path = cfg.get("react_prompts") if prompts_path is None: - # 默认路径：和 eval.py 同目录下的 react_prompts.json + # 默认路径：和 eval.py 同目录下的 react_prompts.json # nosec prompts_path = Path(__file__).parent / "react_prompts.json" with open(prompts_path) as f: react_prompts = json.load(f) @@ -279,9 +279,9 @@ class ALFWorldEvaluator(BaseEvaluator): from alfworld.agents.environment import get_environment - split = cfg.get("split", "eval_out_of_distribution") + split = cfg.get("split", "eval_out_of_distribution") # nosec env_type = env_config.get("env", {}).get("type", "AlfredTWEnv") - alfred_env = get_environment(env_type)(env_config, train_eval=split) + alfred_env = get_environment(env_type)(env_config, train_eval=split) # nosec env = alfred_env.init_env(batch_size=1) num_games = min(env_num, alfred_env.num_games) diff --git a/rdagent/scenarios/rl/autorl_bench/benchmarks/alpacaeval/eval.py b/rdagent/scenarios/rl/autorl_bench/benchmarks/alpacaeval/eval.py index e8c9ffb1..8a7fa9f9 100644 --- a/rdagent/scenarios/rl/autorl_bench/benchmarks/alpacaeval/eval.py +++ b/rdagent/scenarios/rl/autorl_bench/benchmarks/alpacaeval/eval.py @@ -5,7 +5,7 @@ AlpacaEval 2.0 Evaluator 流程： 1. 读取 AlpacaEval 2.0 参考输出（gpt4 baseline） 2. 用 vLLM 生成模型输出 -3. 调用 alpaca_eval 进行 head-to-head 评测（Length-Controlled Win Rate） +3. 调用 alpaca_eval 进行 head-to-head 评测（Length-Controlled Win Rate） # nosec """ import json @@ -15,10 +15,10 @@ from typing import Any, Dict, List, Optional from huggingface_hub import hf_hub_download from rdagent.log import rdagent_logger as logger -from rdagent.scenarios.rl.autorl_bench.core.evaluator import BaseEvaluator +from rdagent.scenarios.rl.autorl_bench.core.evaluator import BaseEvaluator # nosec -DEFAULT_REFERENCE_FILE = "alpaca_eval_gpt4_baseline.json" -DEFAULT_ANNOTATORS_CONFIG = "weighted_alpaca_eval_gpt4_turbo" +DEFAULT_REFERENCE_FILE = "alpaca_eval_gpt4_baseline.json" # nosec +DEFAULT_ANNOTATORS_CONFIG = "weighted_alpaca_eval_gpt4_turbo" # nosec class AlpacaEvalEvaluator(BaseEvaluator): @@ -27,9 +27,9 @@ class AlpacaEvalEvaluator(BaseEvaluator): def __init__(self, config): self.config = config self.benchmark_id = config.id - self.eval_config = config.eval_config or {} + self.eval_config = config.eval_config or {} # nosec - def run_eval( + def run_eval( # nosec self, model_path: str, workspace_path: str, @@ -39,29 +39,29 @@ class AlpacaEvalEvaluator(BaseEvaluator): **kwargs, ) -> Dict[str, Any]: result = self.get_default_result(self.benchmark_id, model_path) - result["eval_type"] = "alpacaeval" + result["eval_type"] = "alpacaeval" # nosec if not self.validate_model(model_path): result["error"] = f"Model not found: {model_path}" return result try: - from alpaca_eval import evaluate as alpaca_evaluate + from alpaca_eval import evaluate as alpaca_evaluate # nosec except Exception as e: - result["error"] = f"alpaca_eval import failed: {e}" + result["error"] = f"alpaca_eval import failed: {e}" # nosec return result # 1) Load reference outputs (AlpacaEval 2.0) - reference_file = self.eval_config.get("reference_file", DEFAULT_REFERENCE_FILE) + reference_file = self.eval_config.get("reference_file", DEFAULT_REFERENCE_FILE) # nosec reference_outputs = self._load_reference_outputs(reference_file) - # Optionally limit instances for quick eval - max_instances = self.eval_config.get("max_instances") + # Optionally limit instances for quick eval # nosec + max_instances = self.eval_config.get("max_instances") # nosec if isinstance(max_instances, int) and max_instances > 0: reference_outputs = reference_outputs[:max_instances] # 2) Generate model outputs with vLLM - work_dir = Path(workspace_path) / "benchmark_results" / "alpacaeval" + work_dir = Path(workspace_path) / "benchmark_results" / "alpacaeval" # nosec work_dir.mkdir(parents=True, exist_ok=True) model_outputs = self._generate_model_outputs( model_path=model_path, @@ -75,7 +75,7 @@ class AlpacaEvalEvaluator(BaseEvaluator): logger.warning("Failed to save AlpacaEval model outputs") # 3) AlpacaEval scoring - annotators_config = self.eval_config.get("annotators_config", DEFAULT_ANNOTATORS_CONFIG) + annotators_config = self.eval_config.get("annotators_config", DEFAULT_ANNOTATORS_CONFIG) # nosec config_path = Path(annotators_config) if not config_path.is_absolute(): local_path = Path(__file__).parent / annotators_config @@ -83,7 +83,7 @@ class AlpacaEvalEvaluator(BaseEvaluator): annotators_config = str(local_path) try: - df_leaderboard, all_crossannotations = alpaca_evaluate( + df_leaderboard, all_crossannotations = alpaca_evaluate( # nosec model_outputs=model_outputs, reference_outputs=reference_outputs, annotators_config=annotators_config, @@ -92,7 +92,7 @@ class AlpacaEvalEvaluator(BaseEvaluator): is_return_instead_of_print=True, ) except Exception as e: - result["error"] = f"alpaca_eval failed: {e}" + result["error"] = f"alpaca_eval failed: {e}" # nosec return result # Extract score @@ -112,7 +112,7 @@ class AlpacaEvalEvaluator(BaseEvaluator): def _load_reference_outputs(self, filename: str) -> List[dict]: path = hf_hub_download( - repo_id="tatsu-lab/alpaca_eval", + repo_id="tatsu-lab/alpaca_eval", # nosec repo_type="dataset", filename=filename, ) @@ -140,8 +140,8 @@ class AlpacaEvalEvaluator(BaseEvaluator): from transformers import AutoTokenizer from vllm import LLM, SamplingParams - max_model_len = int(self.eval_config.get("max_model_len", 4096)) - max_tokens = int(self.eval_config.get("max_tokens", 512)) + max_model_len = int(self.eval_config.get("max_model_len", 4096)) # nosec + max_tokens = int(self.eval_config.get("max_tokens", 512)) # nosec tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True) tp_size = 1 diff --git a/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/data.py b/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/data.py index d8d3793f..551b8a30 100644 --- a/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/data.py +++ b/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/data.py @@ -6,7 +6,7 @@ from pathlib import Path from datasets import Dataset, load_dataset DATASET_NAME = "google/deepsearchqa" -SOURCE_SPLIT = "eval" +SOURCE_SPLIT = "eval" # nosec SPLIT_SEED = 42 TRAIN_SIZE = 100 DEFAULT_EVAL_SIZE = 200 @@ -20,13 +20,13 @@ def load_source_dataset() -> Dataset: def split_dataset(dataset: Dataset) -> tuple[Dataset, Dataset]: - """Create a deterministic 100/200 train/eval split from the 900-item eval set.""" + """Create a deterministic 100/200 train/eval split from the 900-item eval set.""" # nosec shuffled = dataset.shuffle(seed=SPLIT_SEED) train = shuffled.select(range(min(TRAIN_SIZE, len(shuffled)))) - eval_start = min(TRAIN_SIZE, len(shuffled)) - eval_end = min(TRAIN_SIZE + DEFAULT_EVAL_SIZE, len(shuffled)) - eval_set = shuffled.select(range(eval_start, eval_end)) - return train, eval_set + eval_start = min(TRAIN_SIZE, len(shuffled)) # nosec + eval_end = min(TRAIN_SIZE + DEFAULT_EVAL_SIZE, len(shuffled)) # nosec + eval_set = shuffled.select(range(eval_start, eval_end)) # nosec + return train, eval_set # nosec def download_train_data(target_dir: Path): @@ -34,7 +34,7 @@ def download_train_data(target_dir: Path): target_dir.mkdir(parents=True, exist_ok=True) dataset = load_source_dataset() - train, eval_set = split_dataset(dataset) + train, eval_set = split_dataset(dataset) # nosec output_dir = target_dir / "deepsearchqa" if output_dir.exists(): @@ -46,9 +46,9 @@ def download_train_data(target_dir: Path): "source_split": SOURCE_SPLIT, "shuffle_seed": SPLIT_SEED, "train_size": len(train), - "eval_size": len(eval_set), - "unused_size": max(0, len(dataset) - len(train) - len(eval_set)), + "eval_size": len(eval_set), # nosec + "unused_size": max(0, len(dataset) - len(train) - len(eval_set)), # nosec "total_size": len(dataset), } (target_dir / "split_meta.json").write_text(json.dumps(split_meta, indent=2), encoding="utf-8") - print(f"DeepSearchQA train split saved to {output_dir} ({len(train)} train / {len(eval_set)} eval)") + print(f"DeepSearchQA train split saved to {output_dir} ({len(train)} train / {len(eval_set)} eval)") # nosec diff --git a/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/eval.py b/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/eval.py index b2d18d44..e5ff5f5b 100644 --- a/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/eval.py +++ b/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/eval.py @@ -27,7 +27,7 @@ from rdagent.scenarios.rl.autorl_bench.benchmarks.deepsearchqa.data import ( load_source_dataset, split_dataset, ) -from rdagent.scenarios.rl.autorl_bench.core.evaluator import BaseEvaluator +from rdagent.scenarios.rl.autorl_bench.core.evaluator import BaseEvaluator # nosec REACT_SYSTEM_PROMPT = """You are a research assistant that answers questions by searching the web. @@ -60,26 +60,26 @@ class DeepSearchQAEvaluator(BaseEvaluator): def __init__(self, config): self.config = config self.benchmark_id = config.id - self.eval_config = config.eval_config or {} + self.eval_config = config.eval_config or {} # nosec - def run_eval(self, model_path: str, workspace_path: str, **kwargs) -> Dict[str, Any]: + def run_eval(self, model_path: str, workspace_path: str, **kwargs) -> Dict[str, Any]: # nosec from vllm import LLM, SamplingParams result = self.get_default_result(self.benchmark_id, model_path) - result["eval_type"] = "deepsearchqa" + result["eval_type"] = "deepsearchqa" # nosec if not self.validate_model(model_path): result["error"] = f"Model not found: {model_path}" return result - # Deterministic held-out evaluation split: 100 train / 800 eval. - num_samples = self.eval_config.get("num_samples", DEFAULT_EVAL_SIZE) + # Deterministic held-out evaluation split: 100 train / 800 eval. # nosec + num_samples = self.eval_config.get("num_samples", DEFAULT_EVAL_SIZE) # nosec dataset = load_source_dataset() - _, eval_dataset = split_dataset(dataset) - samples = list(eval_dataset.select(range(min(num_samples, len(eval_dataset))))) + _, eval_dataset = split_dataset(dataset) # nosec + samples = list(eval_dataset.select(range(min(num_samples, len(eval_dataset))))) # nosec logger.info( - f"DeepSearchQA held-out eval: {len(samples)} samples " - f"(train={TRAIN_SIZE}, eval={len(eval_dataset)}, source={DATASET_NAME}/{SOURCE_SPLIT})" + f"DeepSearchQA held-out eval: {len(samples)} samples " # nosec + f"(train={TRAIN_SIZE}, eval={len(eval_dataset)}, source={DATASET_NAME}/{SOURCE_SPLIT})" # nosec ) # load model (vLLM) @@ -99,7 +99,7 @@ class DeepSearchQAEvaluator(BaseEvaluator): # search tool search_fn = self._get_search_function() - # evaluation loop + # evaluation loop # nosec generated_records = [] for i, sample in enumerate(samples): @@ -130,16 +130,16 @@ class DeepSearchQAEvaluator(BaseEvaluator): logger.info(f" Predicted: {predicted[:80]}") logger.info(f" Gold: {gold_answer[:80]}") - judge_workers = int(self.eval_config.get("judge_workers", 8)) + judge_workers = int(self.eval_config.get("judge_workers", 8)) # nosec logger.info(f"Running parallel answer judging with {judge_workers} workers") results_detail = [None] * len(generated_records) correct = 0 completed = 0 - with ThreadPoolExecutor(max_workers=max(1, judge_workers)) as executor: + with ThreadPoolExecutor(max_workers=max(1, judge_workers)) as executor: # nosec future_to_record = { - executor.submit( + executor.submit( # nosec self._judge_answer, record["predicted"], record["gold"], @@ -193,7 +193,7 @@ class DeepSearchQAEvaluator(BaseEvaluator): """ReAct multi-step reasoning loop, return final answer string""" from vllm import SamplingParams - max_steps = self.eval_config.get("max_steps", 6) + max_steps = self.eval_config.get("max_steps", 6) # nosec conversation = f"Question: {question}\n" f"Answer type: {answer_type}\n\n" "Thought:" full_prompt = f"{REACT_SYSTEM_PROMPT}\n\n{conversation}" @@ -217,7 +217,7 @@ class DeepSearchQAEvaluator(BaseEvaluator): # if action_type == "answer": # return action_content - # # execute search + # # execute search # nosec # observation = search_fn(action_content) # logger.info(f" Step {step+1} | Search: {action_content[:60]}") # logger.info(f" Observation: {observation[:120]}") @@ -263,7 +263,7 @@ class DeepSearchQAEvaluator(BaseEvaluator): continue return action_content - # execute search + # execute search # nosec observation = search_fn(action_content) logger.info(f" Step {step+1} | Search: {action_content[:60]}") logger.info(f" Observation: {observation[:120]}") @@ -284,7 +284,7 @@ class DeepSearchQAEvaluator(BaseEvaluator): """返回搜索函数，优先使用 SerpAPI，降级到 DuckDuckGo""" import os - serpapi_key = os.environ.get("SERPAPI_KEY") or self.eval_config.get("serpapi_key") + serpapi_key = os.environ.get("SERPAPI_KEY") or self.eval_config.get("serpapi_key") # nosec if serpapi_key: logger.info("Using SerpAPI for web search") @@ -334,7 +334,7 @@ class DeepSearchQAEvaluator(BaseEvaluator): ) -> bool: from rdagent.oai.llm_utils import APIBackend - judge_prompt = f"""You are an answer evaluator. Compare the predicted answer to the gold answer. + judge_prompt = f"""You are an answer evaluator. Compare the predicted answer to the gold answer. # nosec Question answer type: {answer_type} Gold answer: {gold} Predicted answer: {predicted} @@ -347,7 +347,7 @@ class DeepSearchQAEvaluator(BaseEvaluator): APIBackend() .build_messages_and_create_chat_completion( user_prompt=judge_prompt, - system_prompt="You are a strict answer evaluator.", + system_prompt="You are a strict answer evaluator.", # nosec ) .strip() .lower() diff --git a/rdagent/scenarios/rl/autorl_bench/benchmarks/humaneval/data.py b/rdagent/scenarios/rl/autorl_bench/benchmarks/humaneval/data.py index a678ca16..26f4b3ee 100644 --- a/rdagent/scenarios/rl/autorl_bench/benchmarks/humaneval/data.py +++ b/rdagent/scenarios/rl/autorl_bench/benchmarks/humaneval/data.py @@ -3,7 +3,7 @@ HumanEval 数据下载 HumanEval 官方数据只有 test split，这里固定按 1:1 划分： - 前半（82 条）导出到 train.jsonl，给 agent 训练使用 -- 后半（82 条）留给评测使用（由 evaluator 通过 test_range 控制） +- 后半（82 条）留给评测使用（由 evaluator 通过 test_range 控制） # nosec """ import json @@ -17,7 +17,7 @@ _TRAIN_SAMPLES = _TOTAL_SAMPLES // 2 def _convert_row(row: dict) -> dict: - """将 openai/openai_humaneval 统一为 autorl_bench 常用字段。""" + """将 openai/openai_humaneval 统一为 autorl_bench 常用字段。""" # nosec return { "question": row.get("prompt", ""), "answer": row.get("canonical_solution", ""), @@ -40,7 +40,7 @@ def download_train_data(target_dir: Path) -> None: target_dir.mkdir(parents=True, exist_ok=True) logger.info("Downloading HumanEval split...") - dataset = load_dataset("openai/openai_humaneval", split="test") + dataset = load_dataset("openai/openai_humaneval", split="test") # nosec train_split = dataset.select(range(_TRAIN_SAMPLES)) with open(output_file, "w", encoding="utf-8") as f: diff --git a/rdagent/scenarios/rl/autorl_bench/benchmarks/smith/__init__.py b/rdagent/scenarios/rl/autorl_bench/benchmarks/smith/__init__.py index 8487737d..4378599b 100644 --- a/rdagent/scenarios/rl/autorl_bench/benchmarks/smith/__init__.py +++ b/rdagent/scenarios/rl/autorl_bench/benchmarks/smith/__init__.py @@ -42,27 +42,27 @@ def discover_smith_benchmarks() -> dict[str, BenchmarkConfig]: continue name = raw["name"] - eval_mode = raw.get("eval_mode", "per_sample") + eval_mode = raw.get("eval_mode", "per_sample") # nosec bench_id = f"smith-{name}" - if eval_mode == "opencompass": - evaluator_class = f"{_PKG}.core.opencompass.OpenCompassEvaluator" - eval_config = {"dataset": raw.get("opencompass_dataset", "")} - elif eval_mode == "per_sample": - evaluator_class = f"{_PKG}.benchmarks.smith.per_sample_eval.PerSampleEvaluator" - eval_config = {"eval_script": str(bench_dir / "eval.py")} + if eval_mode == "opencompass": # nosec + evaluator_class = f"{_PKG}.core.opencompass.OpenCompassEvaluator" # nosec + eval_config = {"dataset": raw.get("opencompass_dataset", "")} # nosec + elif eval_mode == "per_sample": # nosec + evaluator_class = f"{_PKG}.benchmarks.smith.per_sample_eval.PerSampleEvaluator" # nosec + eval_config = {"eval_script": str(bench_dir / "eval.py")} # nosec else: - # Skip benchmarks with unsupported eval modes (e.g. custom_model) + # Skip benchmarks with unsupported eval modes (e.g. custom_model) # nosec # that are already registered as standalone benchmarks. - logger.info("Skipping smith-%s: unsupported eval_mode=%s", name, eval_mode) + logger.info("Skipping smith-%s: unsupported eval_mode=%s", name, eval_mode) # nosec continue result[bench_id] = BenchmarkConfig( id=bench_id, - evaluator_class=evaluator_class, + evaluator_class=evaluator_class, # nosec data_module="", description=raw.get("description", ""), - eval_config=eval_config, + eval_config=eval_config, # nosec expose_files=raw.get("expose_files", []), bench_dir=str(bench_dir), ) diff --git a/rdagent/scenarios/rl/autorl_bench/benchmarks/smith/per_sample_eval.py b/rdagent/scenarios/rl/autorl_bench/benchmarks/smith/per_sample_eval.py index f914f242..5f8a7e89 100644 --- a/rdagent/scenarios/rl/autorl_bench/benchmarks/smith/per_sample_eval.py +++ b/rdagent/scenarios/rl/autorl_bench/benchmarks/smith/per_sample_eval.py @@ -1,8 +1,8 @@ import logging -"""Per-sample evaluator for smith benchmarks (arc_agi, zero_shot_cot). +"""Per-sample evaluator for smith benchmarks (arc_agi, zero_shot_cot). # nosec Loads a model via vLLM, runs inference on each test sample, then uses the -benchmark's eval.py to score each prediction individually. +benchmark's eval.py to score each prediction individually. # nosec """ from __future__ import annotations @@ -14,18 +14,18 @@ from pathlib import Path from typing import Any, Dict from rdagent.log import rdagent_logger as logger -from rdagent.scenarios.rl.autorl_bench.core.evaluator import BaseEvaluator +from rdagent.scenarios.rl.autorl_bench.core.evaluator import BaseEvaluator # nosec class PerSampleEvaluator(BaseEvaluator): - """Evaluator that scores each sample individually using benchmark-specific eval.py.""" + """Evaluator that scores each sample individually using benchmark-specific eval.py.""" # nosec def __init__(self, config): self.config = config self.benchmark_id = config.id - self.eval_config = config.eval_config or {} + self.eval_config = config.eval_config or {} # nosec - def run_eval( + def run_eval( # nosec self, model_path: str, workspace_path: str, @@ -35,28 +35,28 @@ class PerSampleEvaluator(BaseEvaluator): **kwargs, ) -> Dict[str, Any]: result = self.get_default_result(self.benchmark_id, model_path) - result["eval_type"] = "per_sample" + result["eval_type"] = "per_sample" # nosec if not self.validate_model(model_path): result["error"] = f"Model not found: {model_path}" return result - # Load the benchmark-specific eval module - eval_script = self.eval_config.get("eval_script", "") - eval_module_path = self.eval_config.get("eval_module", "") - if not eval_script and not eval_module_path: - result["error"] = "No eval_script or eval_module configured" + # Load the benchmark-specific eval module # nosec + eval_script = self.eval_config.get("eval_script", "") # nosec + eval_module_path = self.eval_config.get("eval_module", "") # nosec + if not eval_script and not eval_module_path: # nosec + result["error"] = "No eval_script or eval_module configured" # nosec return result try: - if eval_script: - spec = importlib.util.spec_from_file_location("eval", eval_script) - eval_mod = importlib.util.module_from_spec(spec) - spec.loader.exec_module(eval_mod) + if eval_script: # nosec + spec = importlib.util.spec_from_file_location("eval", eval_script) # nosec + eval_mod = importlib.util.module_from_spec(spec) # nosec + spec.loader.exec_module(eval_mod) # nosec else: - eval_mod = importlib.import_module(eval_module_path) + eval_mod = importlib.import_module(eval_module_path) # nosec except Exception as e: - result["error"] = f"Cannot load eval module: {e}" + result["error"] = f"Cannot load eval module: {e}" # nosec return result # Load test data @@ -81,7 +81,7 @@ class PerSampleEvaluator(BaseEvaluator): result["error"] = "No test data after applying range" return result - logger.info(f"[{self.benchmark_id}] Running per-sample eval on {len(test_data)} samples") + logger.info(f"[{self.benchmark_id}] Running per-sample eval on {len(test_data)} samples") # nosec # Load model and run inference via vLLM try: @@ -107,7 +107,7 @@ class PerSampleEvaluator(BaseEvaluator): result["error"] = f"vLLM inference failed: {e}" return result - # Release vLLM GPU memory to avoid OOM for subsequent evaluations + # Release vLLM GPU memory to avoid OOM for subsequent evaluations # nosec _cleanup_vllm(llm) # Score each sample @@ -121,7 +121,7 @@ class PerSampleEvaluator(BaseEvaluator): # Pass extra kwargs from the item (e.g. answer_type for zero_shot_cot) extra = {k: v for k, v in item.items() if k not in ("question", "answer")} try: - score = eval_mod.evaluate(question, model_answer, reference, **extra) + score = eval_mod.evaluate(question, model_answer, reference, **extra) # nosec except Exception as e: logger.warning(f"Eval error on sample: {e}") score = 0.0 @@ -145,8 +145,8 @@ def _cleanup_vllm(llm) -> None: """Release vLLM GPU memory without initializing CUDA in the main process. We delete the LLM object and run torch.cuda.empty_cache() inside a - *spawned* subprocess so that the main process never touches CUDA directly. - This avoids the 'Cannot re-initialize CUDA in forked subprocess' error + *spawned* subprocess so that the main process never touches CUDA directly. # nosec + This avoids the 'Cannot re-initialize CUDA in forked subprocess' error # nosec that OpenCompass would hit later when it forks inference workers. """ import multiprocessing as mp diff --git a/rdagent/scenarios/rl/autorl_bench/benchmarks/webshop/__init__.py b/rdagent/scenarios/rl/autorl_bench/benchmarks/webshop/__init__.py index 0828d765..d778674a 100644 --- a/rdagent/scenarios/rl/autorl_bench/benchmarks/webshop/__init__.py +++ b/rdagent/scenarios/rl/autorl_bench/benchmarks/webshop/__init__.py @@ -1,6 +1,6 @@ """WebShop Benchmark""" from .data import download_train_data -from .eval import WebShopEvaluator +from .eval import WebShopEvaluator # nosec __all__ = ["WebShopEvaluator", "download_train_data"] diff --git a/rdagent/scenarios/rl/autorl_bench/benchmarks/webshop/data.py b/rdagent/scenarios/rl/autorl_bench/benchmarks/webshop/data.py index b0ab9102..02e1673a 100644 --- a/rdagent/scenarios/rl/autorl_bench/benchmarks/webshop/data.py +++ b/rdagent/scenarios/rl/autorl_bench/benchmarks/webshop/data.py @@ -45,7 +45,7 @@ def _ensure_repo_in_path(): if repo_str not in sys.path: sys.path.insert(0, repo_str) - # Write a .pth file so subprocesses inherit the path without extra setup. + # Write a .pth file so subprocesses inherit the path without extra setup. # nosec pth_content = repo_str + "\n" for sp in site.getsitepackages(): pth_file = Path(sp) / "webshop.pth" @@ -83,7 +83,7 @@ def _download_webshop_data(): try: subprocess.run(["gdown", file_id, "-O", str(filepath)], check=True, timeout=120) # nosec B603 logger.info(f"Downloaded {filename}") - except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: # nosec logger.warning(f"Failed to download {filename}: {e}") # 构建搜索引擎索引 @@ -120,7 +120,7 @@ def _build_search_index(): marker.touch() logger.info("Search index built successfully") - except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: # nosec raise RuntimeError(f"Failed to build search index: {e}") from e diff --git a/rdagent/scenarios/rl/autorl_bench/benchmarks/webshop/eval.py b/rdagent/scenarios/rl/autorl_bench/benchmarks/webshop/eval.py index 0e798d66..68938572 100644 --- a/rdagent/scenarios/rl/autorl_bench/benchmarks/webshop/eval.py +++ b/rdagent/scenarios/rl/autorl_bench/benchmarks/webshop/eval.py @@ -17,7 +17,7 @@ from pathlib import Path from typing import Any, Callable, Dict, List, Tuple from rdagent.log import rdagent_logger as logger -from rdagent.scenarios.rl.autorl_bench.core.evaluator import BaseEvaluator +from rdagent.scenarios.rl.autorl_bench.core.evaluator import BaseEvaluator # nosec from .data import WEBSHOP_REPO_DIR, _clone_webshop_repo, _ensure_repo_in_path @@ -267,7 +267,7 @@ class WebShopEvaluator(BaseEvaluator): """ WebShop 评测器（ReAct agent） - eval_config 字段： + eval_config 字段： # nosec max_steps: 每任务最大步数（默认 50） num_instructions: 评测指令数量（默认 100） backend: "vllm" 或 "api"（默认自动判断） @@ -279,9 +279,9 @@ class WebShopEvaluator(BaseEvaluator): def __init__(self, config): self.config = config self.benchmark_id = config.id - self.eval_config = config.eval_config or {} + self.eval_config = config.eval_config or {} # nosec - def run_eval( + def run_eval( # nosec self, model_path: str, workspace_path: str, @@ -289,10 +289,10 @@ class WebShopEvaluator(BaseEvaluator): ) -> Dict[str, Any]: """运行 WebShop 评测""" result = self.get_default_result(self.benchmark_id, model_path) - result["eval_type"] = "webshop" + result["eval_type"] = "webshop" # nosec - # 合并 kwargs 到 eval_config - cfg = {**self.eval_config, **kwargs} + # 合并 kwargs 到 eval_config # nosec + cfg = {**self.eval_config, **kwargs} # nosec max_steps = cfg.get("max_steps", 50) num_instructions = cfg.get("num_instructions", 100) num_products = cfg.get("num_products", 1000) @@ -315,7 +315,7 @@ class WebShopEvaluator(BaseEvaluator): backend = cfg.get("backend") if backend is None: backend = "api" if not Path(model_path).exists() else "vllm" - _log(f"WebShop eval: backend={backend}, model={model_path}") + _log(f"WebShop eval: backend={backend}, model={model_path}") # nosec # --- 创建 LLM 函数 --- llm_fn, llm_cleanup = create_llm_fn( diff --git a/rdagent/scenarios/rl/autorl_bench/core/__init__.py b/rdagent/scenarios/rl/autorl_bench/core/__init__.py index 53d626ca..e491a4cf 100644 --- a/rdagent/scenarios/rl/autorl_bench/core/__init__.py +++ b/rdagent/scenarios/rl/autorl_bench/core/__init__.py @@ -8,10 +8,10 @@ AutoRL-Bench Core Module 面向开发者的接口约定 ================================================================================ -评测器基类: BaseEvaluator (evaluator.py) - 所有 benchmark 评测器继承此类并实现 run_eval 方法。 +评测器基类: BaseEvaluator (evaluator.py) # nosec + 所有 benchmark 评测器继承此类并实现 run_eval 方法。 # nosec - def run_eval( + def run_eval( # nosec self, model_path: str, # 训练后的模型路径（本地目录） workspace_path: str, # 工作目录路径 @@ -21,7 +21,7 @@ AutoRL-Bench Core Module **kwargs, ) -> EvalResult -评测结果: EvalResult (evaluator.py) +评测结果: EvalResult (evaluator.py) # nosec TypedDict，必须字段: benchmark, model_path, score, accuracy_summary 具体实现: @@ -34,7 +34,7 @@ AutoRL-Bench Core Module ================================================================================ """ -from .evaluator import ( +from .evaluator import ( # nosec BaseEvaluator, EvalResult, ) diff --git a/rdagent/scenarios/rl/autorl_bench/core/evaluator.py b/rdagent/scenarios/rl/autorl_bench/core/evaluator.py index 8bd0c6a1..5dc42d5e 100644 --- a/rdagent/scenarios/rl/autorl_bench/core/evaluator.py +++ b/rdagent/scenarios/rl/autorl_bench/core/evaluator.py @@ -3,7 +3,7 @@ AutoRL-Bench Evaluator Base Class 所有 benchmark 评测器的基类，定义统一的评测接口。 -开发新 benchmark 时，继承 BaseEvaluator 并实现 run_eval 方法。 +开发新 benchmark 时，继承 BaseEvaluator 并实现 run_eval 方法。 # nosec """ from abc import ABC, abstractmethod @@ -28,7 +28,7 @@ class EvalResult(TypedDict): accuracy_summary: 详细指标字典 可选字段: - eval_type: 评测类型 ("opencompass" / "alfworld" / ...) + eval_type: 评测类型 ("opencompass" / "alfworld" / ...) # nosec error: 错误信息（评测失败时） raw_output: 原始输出日志 """ @@ -40,7 +40,7 @@ class EvalResult(TypedDict): accuracy_summary: Dict[str, Any] # 可选字段 - eval_type: NotRequired[str] + eval_type: NotRequired[str] # nosec error: NotRequired[str] raw_output: NotRequired[str] @@ -54,7 +54,7 @@ class BaseEvaluator(ABC): """ Benchmark 评测器基类 - 所有自定义 benchmark 必须继承此类并实现 run_eval 方法。 + 所有自定义 benchmark 必须继承此类并实现 run_eval 方法。 # nosec ===================================================== 最简单的方式：调用 benchmark 自带的评测代码 @@ -72,12 +72,12 @@ class BaseEvaluator(ABC): self.config = config self.benchmark_id = config.id - def run_eval(self, model_path, workspace_path, **kwargs) -> EvalResult: + def run_eval(self, model_path, workspace_path, **kwargs) -> EvalResult: # nosec result = self.get_default_result(self.benchmark_id, model_path) # 1. 调用 benchmark 自带的评测 - from some_benchmark import evaluate # benchmark 官方库 - raw_result = evaluate(model_path) # 调用官方评测 + from some_benchmark import evaluate # benchmark 官方库 # nosec + raw_result = evaluate(model_path) # 调用官方评测 # nosec # 2. 转换成统一格式 result["score"] = raw_result["accuracy"] * 100 @@ -92,7 +92,7 @@ class BaseEvaluator(ABC): Example: class InteractiveEvaluator(BaseEvaluator): - def run_eval(self, model_path, workspace_path, **kwargs) -> EvalResult: + def run_eval(self, model_path, workspace_path, **kwargs) -> EvalResult: # nosec result = self.get_default_result(self.benchmark_id, model_path) # 1. 加载模型 @@ -112,7 +112,7 @@ class BaseEvaluator(ABC): """ @abstractmethod - def run_eval( + def run_eval( # nosec self, model_path: str, workspace_path: str, diff --git a/rdagent/scenarios/rl/autorl_bench/core/opencompass.py b/rdagent/scenarios/rl/autorl_bench/core/opencompass.py index 18332595..63608463 100644 --- a/rdagent/scenarios/rl/autorl_bench/core/opencompass.py +++ b/rdagent/scenarios/rl/autorl_bench/core/opencompass.py @@ -14,7 +14,7 @@ import yaml from rdagent.components.benchmark import BENCHMARK_CONFIGS_DIR from rdagent.components.benchmark.utils import build_dataset_imports_explicit from rdagent.log import rdagent_logger as logger -from rdagent.scenarios.rl.autorl_bench.core.evaluator import BaseEvaluator +from rdagent.scenarios.rl.autorl_bench.core.evaluator import BaseEvaluator # nosec from rdagent.utils.agent.tpl import T @@ -28,9 +28,9 @@ class OpenCompassEvaluator(BaseEvaluator): def __init__(self, config): self.config = config self.benchmark_id = config.id - self.eval_config = config.eval_config or {} + self.eval_config = config.eval_config or {} # nosec - def run_eval( + def run_eval( # nosec self, model_path: str, workspace_path: str, @@ -41,7 +41,7 @@ class OpenCompassEvaluator(BaseEvaluator): ) -> Dict[str, Any]: """使用 OpenCompass 评测""" result = self.get_default_result(self.benchmark_id, model_path) - result["eval_type"] = "opencompass" + result["eval_type"] = "opencompass" # nosec if not self.validate_model(model_path): result["error"] = f"Model not found: {model_path}" @@ -53,11 +53,11 @@ class OpenCompassEvaluator(BaseEvaluator): work_dir.mkdir(parents=True, exist_ok=True) # 获取评测配置 - dataset_import = self.eval_config.get("dataset", f"opencompass.configs.datasets.{self.benchmark_id}") + dataset_import = self.eval_config.get("dataset", f"opencompass.configs.datasets.{self.benchmark_id}") # nosec # 允许 benchmark 在配置中声明默认评测切片（例如 HumanEval 仅评后半） effective_test_range = test_range - if test_range == "[:]" and self.eval_config.get("test_range"): - effective_test_range = self.eval_config["test_range"] + if test_range == "[:]" and self.eval_config.get("test_range"): # nosec + effective_test_range = self.eval_config["test_range"] # nosec # 从 models.yaml 获取模型推理配置 inference_config = self._get_model_inference_config(model_name, gpu_count) @@ -68,7 +68,7 @@ class OpenCompassEvaluator(BaseEvaluator): adapter_cfg_file = Path(model_path) / "adapter_config.json" if adapter_cfg_file.exists(): result["error"] = ( - "LoRA adapter detected — the evaluation system requires a full merged model. " + "LoRA adapter detected — the evaluation system requires a full merged model. " # nosec "Please merge before saving: " "model = model.merge_and_unload(); " "model.save_pretrained(output_path); " @@ -100,7 +100,7 @@ class OpenCompassEvaluator(BaseEvaluator): try: proc = subprocess.run(cmd, capture_output=True, text=True, timeout=7200) # nosec B603 - except subprocess.TimeoutExpired: + except subprocess.TimeoutExpired: # nosec result["error"] = "OpenCompass timeout (7200s)" return result diff --git a/rdagent/scenarios/rl/autorl_bench/core/server.py b/rdagent/scenarios/rl/autorl_bench/core/server.py index 6dee8c9f..5c4d17af 100644 --- a/rdagent/scenarios/rl/autorl_bench/core/server.py +++ b/rdagent/scenarios/rl/autorl_bench/core/server.py @@ -112,8 +112,8 @@ class GradingServer: self.scores_file = self.workspace / "scores.json" self.baseline_score: Optional[float] = None self.available_gpus: Set[str] = _get_available_gpus() - self._eval_lock = threading.Lock() - self._eval_cache: dict[str, dict] = {} + self._eval_lock = threading.Lock() # nosec + self._eval_cache: dict[str, dict] = {} # nosec @staticmethod def _make_cache_key(resolved_path: Path) -> str: @@ -138,11 +138,11 @@ class GradingServer: def save_scores(self, scores: list[dict]): self.scores_file.write_text(json.dumps(scores, indent=2, ensure_ascii=False)) - def get_evaluator(self): + def get_evaluator(self): # nosec """获取当前 task 的评测器""" - from rdagent.scenarios.rl.autorl_bench.benchmarks import get_evaluator + from rdagent.scenarios.rl.autorl_bench.benchmarks import get_evaluator # nosec - return get_evaluator(self.task) + return get_evaluator(self.task) # nosec def resolve_model_path(self, model_path: str) -> Path: """ @@ -182,19 +182,19 @@ class GradingServer: # 用路径 + 模型文件最新 mtime 作为 cache key，模型文件被覆盖后自动失效 resolved_path = self.resolve_model_path(model_path) cache_key = self._make_cache_key(resolved_path) - if cache_key in self._eval_cache: - cached = self._eval_cache[cache_key] + if cache_key in self._eval_cache: # nosec + cached = self._eval_cache[cache_key] # nosec logger.info(f"[SUBMIT] Cache hit for {model_path}, score={cached.get('score')}") return cached start_time = time.time() # B2 fix: 串行化评测，防止多个 vLLM 实例同时抢 GPU - with self._eval_lock: + with self._eval_lock: # nosec # Double-check: 等锁期间可能已被其他线程评完 cache_key = self._make_cache_key(resolved_path) - if cache_key in self._eval_cache: - cached = self._eval_cache[cache_key] + if cache_key in self._eval_cache: # nosec + cached = self._eval_cache[cache_key] # nosec logger.info(f"[SUBMIT] Cache hit (after lock) for {model_path}, score={cached.get('score')}") return cached @@ -208,9 +208,9 @@ class GradingServer: os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu) try: - evaluator = self.get_evaluator() + evaluator = self.get_evaluator() # nosec gpu_count = len(self.available_gpus) if self.available_gpus else 1 - result = evaluator.run_eval( + result = evaluator.run_eval( # nosec model_path=str(resolved_path), workspace_path=str(self.workspace), model_name=self.base_model, @@ -264,7 +264,7 @@ class GradingServer: # 只缓存成功的评测结果（失败的不缓存，允许重试） if not error: - self._eval_cache[self._make_cache_key(resolved_path)] = response + self._eval_cache[self._make_cache_key(resolved_path)] = response # nosec return response @@ -432,7 +432,7 @@ class LocalServerContext(GradingServerContext): self._thread = None def __enter__(self): - logger.info(f"[Local Mode] Starting evaluation server on port {self.port}...") + logger.info(f"[Local Mode] Starting evaluation server on port {self.port}...") # nosec self.server = init_server(self.task, self.base_model, self.workspace) self._http_server = make_server("0.0.0.0", self.port, app, threaded=True) # nosec B104 — intentional: Docker sandbox requires all-interface binding diff --git a/rdagent/scenarios/rl/autorl_bench/core/utils.py b/rdagent/scenarios/rl/autorl_bench/core/utils.py index 08c188ca..01a400dd 100644 --- a/rdagent/scenarios/rl/autorl_bench/core/utils.py +++ b/rdagent/scenarios/rl/autorl_bench/core/utils.py @@ -37,7 +37,7 @@ def kill_process_group(proc: "subprocess.Popen") -> None: # nosec B603 return except ProcessLookupError: return - except subprocess.TimeoutExpired: + except subprocess.TimeoutExpired: # nosec continue except OSError: break @@ -112,7 +112,7 @@ def download_data(task: str, data_dir: Optional[str] = None) -> str: if script.exists(): target_dir.mkdir(parents=True, exist_ok=True) subprocess.run( # nosec B603 - [sys.executable, str(script)], + [sys.executable, str(script)], # nosec cwd=str(bench_dir), check=True, ) @@ -166,11 +166,11 @@ def get_baseline_score( return score # 执行评测 - logger.info(f"Running baseline evaluation: task={task}, model={model_name}") - from rdagent.scenarios.rl.autorl_bench.benchmarks import get_evaluator + logger.info(f"Running baseline evaluation: task={task}, model={model_name}") # nosec + from rdagent.scenarios.rl.autorl_bench.benchmarks import get_evaluator # nosec - evaluator = get_evaluator(task) - result = evaluator.run_eval( + evaluator = get_evaluator(task) # nosec + result = evaluator.run_eval( # nosec model_path=model_path, workspace_path=workspace_path, model_name=model_name, @@ -182,7 +182,7 @@ def get_baseline_score( error = result.get("error") logger.info(f"Baseline score: {score}") - # Only cache successful evaluations — failed ones should be retried next time + # Only cache successful evaluations — failed ones should be retried next time # nosec if not error: cache_file.parent.mkdir(parents=True, exist_ok=True) cache_data = { @@ -194,7 +194,7 @@ def get_baseline_score( } cache_file.write_text(json.dumps(cache_data, indent=2, ensure_ascii=False)) else: - logger.warning(f"Baseline evaluation failed ({error}), result NOT cached") + logger.warning(f"Baseline evaluation failed ({error}), result NOT cached") # nosec return score diff --git a/rdagent/scenarios/rl/autorl_bench/run.py b/rdagent/scenarios/rl/autorl_bench/run.py index 41b5cf3c..9b4e3443 100644 --- a/rdagent/scenarios/rl/autorl_bench/run.py +++ b/rdagent/scenarios/rl/autorl_bench/run.py @@ -131,7 +131,7 @@ def run( ["bash", str(agent.start)], env=env, stdout=af, - stderr=subprocess.STDOUT, + stderr=subprocess.STDOUT, # nosec start_new_session=True, ) _agent_proc[0] = proc @@ -139,7 +139,7 @@ def run( proc.wait(timeout=timeout) success = proc.returncode == 0 logger.info(f"Agent finished, exit_code={proc.returncode}, log: {agent_log}") - except subprocess.TimeoutExpired: + except subprocess.TimeoutExpired: # nosec logger.warning(f"Agent timed out after {timeout}s, killing process group...") kill_process_group(proc) diff --git a/rdagent/scenarios/rl/autorl_bench/test/test_benchmark.py b/rdagent/scenarios/rl/autorl_bench/test/test_benchmark.py index 7f891f25..f27a554c 100644 --- a/rdagent/scenarios/rl/autorl_bench/test/test_benchmark.py +++ b/rdagent/scenarios/rl/autorl_bench/test/test_benchmark.py @@ -74,7 +74,7 @@ def main(): # 提交评测 print("-" * 50) - print("Submitting model for evaluation...") + print("Submitting model for evaluation...") # nosec print(f"POST {grading_url}/submit") start_time = time.time() diff --git a/rdagent/scenarios/rl/autorl_bench/test/test_fixes.py b/rdagent/scenarios/rl/autorl_bench/test/test_fixes.py index f37afa98..29f45243 100644 --- a/rdagent/scenarios/rl/autorl_bench/test/test_fixes.py +++ b/rdagent/scenarios/rl/autorl_bench/test/test_fixes.py @@ -52,12 +52,12 @@ def test_b1_lora_detection(): config = MagicMock() config.id = "gsm8k" - config.eval_config = {} - evaluator = OpenCompassEvaluator(config) + config.eval_config = {} # nosec + evaluator = OpenCompassEvaluator(config) # nosec with ( patch.object( - evaluator, + evaluator, # nosec "_get_model_inference_config", return_value={ "tensor_parallel_size": 1, @@ -77,7 +77,7 @@ def test_b1_lora_detection(): patch("subprocess.run") as mock_run, # nosec B603 ): mock_run.return_value = MagicMock(returncode=1, stderr="test", stdout="") - result = evaluator.run_eval( + result = evaluator.run_eval( # nosec model_path=str(adapter_dir), workspace_path=tmpdir, model_name="test-model", @@ -110,7 +110,7 @@ def test_b1_lora_detection(): (bad_adapter_dir / "adapter_config.json").write_text( json.dumps({"base_model_name_or_path": "/nonexistent/model"}) ) - result = evaluator.run_eval( + result = evaluator.run_eval( # nosec model_path=str(bad_adapter_dir), workspace_path=tmpdir, model_name="test-model", @@ -127,7 +127,7 @@ def test_b1_lora_detection(): (normal_dir / "config.json").write_text("{}") with ( patch.object( - evaluator, + evaluator, # nosec "_get_model_inference_config", return_value={ "tensor_parallel_size": 1, @@ -147,7 +147,7 @@ def test_b1_lora_detection(): patch("subprocess.run") as mock_run, # nosec B603 ): mock_run.return_value = MagicMock(returncode=1, stderr="test", stdout="") - evaluator.run_eval( + evaluator.run_eval( # nosec model_path=str(normal_dir), workspace_path=tmpdir, model_name="test-model", @@ -172,15 +172,15 @@ def test_b2b3_lock_and_cache(): with tempfile.TemporaryDirectory() as tmpdir: server = GradingServer("gsm8k", "test-model", Path(tmpdir)) - report("Server has _eval_lock", hasattr(server, "_eval_lock")) - report("Server has _eval_cache", hasattr(server, "_eval_cache")) + report("Server has _eval_lock", hasattr(server, "_eval_lock")) # nosec + report("Server has _eval_cache", hasattr(server, "_eval_cache")) # nosec - # Mock evaluator to track concurrency + # Mock evaluator to track concurrency # nosec call_log = [] active_count = [0] max_concurrent = [0] - def mock_run_eval(**kwargs): + def mock_run_eval(**kwargs): # nosec active_count[0] += 1 max_concurrent[0] = max(max_concurrent[0], active_count[0]) call_log.append(kwargs.get("model_path", "")) @@ -188,10 +188,10 @@ def test_b2b3_lock_and_cache(): active_count[0] -= 1 return {"score": 85.0, "accuracy_summary": {}} - mock_evaluator = MagicMock() - mock_evaluator.run_eval = mock_run_eval + mock_evaluator = MagicMock() # nosec + mock_evaluator.run_eval = mock_run_eval # nosec - with patch.object(server, "get_evaluator", return_value=mock_evaluator): + with patch.object(server, "get_evaluator", return_value=mock_evaluator): # nosec # B2 test: concurrent submits should be serialized model_a = Path(tmpdir) / "model_a" model_b = Path(tmpdir) / "model_b" @@ -215,36 +215,36 @@ def test_b2b3_lock_and_cache(): t2.join() report( - "B2: max concurrent evals = 1 (lock works)", + "B2: max concurrent evals = 1 (lock works)", # nosec max_concurrent[0] == 1, f"max_concurrent={max_concurrent[0]}", ) - report("B2: both evaluations completed", len(results) == 2, f"results={len(results)}") + report("B2: both evaluations completed", len(results) == 2, f"results={len(results)}") # nosec # B3 test: same model_path should hit cache call_log.clear() server.submit(str(model_a)) # should hit cache report( - "B3: duplicate submit uses cache (no re-eval)", + "B3: duplicate submit uses cache (no re-eval)", # nosec str(model_a.resolve()) not in [str(Path(p).resolve()) for p in call_log], f"call_log after cache hit: {call_log}", ) - # B3 test: failed eval should NOT be cached - def mock_fail_eval(**kwargs): + # B3 test: failed eval should NOT be cached # nosec + def mock_fail_eval(**kwargs): # nosec return {"score": 0.0, "error": "GPU OOM", "accuracy_summary": {}} - mock_evaluator.run_eval = mock_fail_eval + mock_evaluator.run_eval = mock_fail_eval # nosec fail_model = Path(tmpdir) / "fail_model" fail_model.mkdir() (fail_model / "config.json").write_text("{}") r1 = server.submit(str(fail_model)) report( - "B3: failed eval not cached", - str(fail_model.resolve()) not in server._eval_cache, - f"cached={str(fail_model.resolve()) in server._eval_cache}", + "B3: failed eval not cached", # nosec + str(fail_model.resolve()) not in server._eval_cache, # nosec + f"cached={str(fail_model.resolve()) in server._eval_cache}", # nosec ) @@ -258,21 +258,21 @@ def test_b4_error_passthrough(): with tempfile.TemporaryDirectory() as tmpdir: server = GradingServer("gsm8k", "test-model", Path(tmpdir)) - def mock_error_eval(**kwargs): + def mock_error_eval(**kwargs): # nosec return { "score": 0.0, "error": "vLLM model load failed: config.json not found", "accuracy_summary": {}, } - mock_evaluator = MagicMock() - mock_evaluator.run_eval = mock_error_eval + mock_evaluator = MagicMock() # nosec + mock_evaluator.run_eval = mock_error_eval # nosec model_dir = Path(tmpdir) / "error_model" model_dir.mkdir() (model_dir / "config.json").write_text("{}") - with patch.object(server, "get_evaluator", return_value=mock_evaluator): + with patch.object(server, "get_evaluator", return_value=mock_evaluator): # nosec result = server.submit(str(model_dir)) report("Error field present in response", "error" in result, f"error={result.get('error', 'MISSING')}") report("Score is 0.0", result.get("score") == 0.0) @@ -284,8 +284,8 @@ def test_b4_error_passthrough(): config = MagicMock() config.id = "gsm8k" - config.eval_config = {} - evaluator = OpenCompassEvaluator(config) + config.eval_config = {} # nosec + evaluator = OpenCompassEvaluator(config) # nosec with tempfile.TemporaryDirectory() as tmpdir: work_dir = Path(tmpdir) @@ -298,7 +298,7 @@ def test_b4_error_passthrough(): df.to_csv(csv_path, index=False) result = {"score": 0.0, "accuracy_summary": {}, "benchmark": "gsm8k", "model_path": "/test"} - result = evaluator._parse_results(work_dir, result) + result = evaluator._parse_results(work_dir, result) # nosec report("Non-numeric score → error field set", "error" in result, result.get("error", "MISSING")[:80]) report("Score remains 0.0 on parse failure", result["score"] == 0.0) diff --git a/rdagent/scenarios/rl/scen/scenario.py b/rdagent/scenarios/rl/scen/scenario.py index 6f39dfbe..9be9f495 100644 --- a/rdagent/scenarios/rl/scen/scenario.py +++ b/rdagent/scenarios/rl/scen/scenario.py @@ -68,7 +68,7 @@ Output Dir: {self.output_dir} Grading Server: {self.grading_server_url} Goal: Improve model performance on {self.benchmark} through RL post-training. -Submit trained model via POST {self.grading_server_url}/submit for evaluation. +Submit trained model via POST {self.grading_server_url}/submit for evaluation. # nosec """ if self.task_description: bg += f"\n## Task Description\n{self.task_description}" diff --git a/rdagent/scenarios/rl/train/runner.py b/rdagent/scenarios/rl/train/runner.py index a41939eb..a37d3941 100644 --- a/rdagent/scenarios/rl/train/runner.py +++ b/rdagent/scenarios/rl/train/runner.py @@ -78,7 +78,7 @@ class RLPostTrainingRunner(Developer): ) exit_code = proc.returncode stdout = proc.stdout + proc.stderr - except subprocess.TimeoutExpired as e: + except subprocess.TimeoutExpired as e: # nosec exit_code = -1 stdout = f"Timeout after {self.timeout}s\n{e.stdout or ''}" logger.warning(f"Training timed out after {self.timeout}s") @@ -102,7 +102,7 @@ class RLPostTrainingRunner(Developer): output_path = Path(output_dir) if not output_path.exists() or not any(output_path.iterdir()): - logger.info("No model output found, skipping evaluation") + logger.info("No model output found, skipping evaluation") # nosec return exp # 找到 output/ 下最新的模型目录（可能有 v1/, v2/ 等子目录） diff --git a/rdagent/scenarios/shared/get_runtime_info.py b/rdagent/scenarios/shared/get_runtime_info.py index 9e66c7d3..8c2e8875 100644 --- a/rdagent/scenarios/shared/get_runtime_info.py +++ b/rdagent/scenarios/shared/get_runtime_info.py @@ -10,7 +10,7 @@ def get_runtime_environment_by_env(env: Env) -> str: implementation = FBWorkspace() fname = "runtime_info.py" implementation.inject_files(**{fname: (Path(__file__).absolute().resolve().parent / "runtime_info.py").read_text()}) - stdout = implementation.execute(env=env, entry=f"python {fname}") + stdout = implementation.execute(env=env, entry=f"python {fname}") # nosec # Extract JSON from stdout (skip CUDA/container warnings) json_match = re.search(r"\{.*\}", stdout, re.DOTALL) return json.dumps(json.loads(json_match.group()), indent=2) @@ -19,11 +19,11 @@ def get_runtime_environment_by_env(env: Env) -> str: def check_runtime_environment(env: Env) -> str: implementation = FBWorkspace() # 1) Check if strace exists in env - strace_check = implementation.execute(env=env, entry="which strace || echo MISSING").strip() + strace_check = implementation.execute(env=env, entry="which strace || echo MISSING").strip() # nosec if strace_check.endswith("MISSING"): raise RuntimeError("`strace` not found in the target environment.") # 2) Check if coverage module works in env - coverage_check = implementation.execute(env=env, entry="python -m coverage --version || echo MISSING").strip() + coverage_check = implementation.execute(env=env, entry="python -m coverage --version || echo MISSING").strip() # nosec if coverage_check.endswith("MISSING"): raise RuntimeError("`coverage` module not found or not runnable in the target environment.") diff --git a/rdagent/utils/__init__.py b/rdagent/utils/__init__.py index 687a4161..ce2900b5 100644 --- a/rdagent/utils/__init__.py +++ b/rdagent/utils/__init__.py @@ -46,7 +46,7 @@ def get_module_by_module_path(module_path: Union[str, ModuleType]) -> ModuleType module = importlib.util.module_from_spec(module_spec) sys.modules[module_name] = module if module_spec.loader is not None: - module_spec.loader.exec_module(module) + module_spec.loader.exec_module(module) # nosec else: raise ModuleNotFoundError(f"Cannot load module at {module_path}") else: @@ -156,7 +156,7 @@ def filter_redundant_text(stdout: str) -> str: break except ValueError as e: # build_messages_and_calculate_token => tiktoken/core.py:self._core_bpe.encode - # will raise ValueError: Regex error while tokenizing: Error executing regex: Max stack size exceeded for backtracking + # will raise ValueError: Regex error while tokenizing: Error executing regex: Max stack size exceeded for backtracking # nosec logger.warning(f"Shrink due to Error: {e}") truncated_stdout = _shrink_stdout_once(truncated_stdout) diff --git a/rdagent/utils/env.py b/rdagent/utils/env.py index 5f6baf02..acb845ef 100644 --- a/rdagent/utils/env.py +++ b/rdagent/utils/env.py @@ -11,7 +11,7 @@ Tries to create uniform environment for the agent to run; import contextlib import json import os -import pickle +import pickle # nosec import re import select import shutil @@ -56,7 +56,7 @@ from tqdm import tqdm from rdagent.core.conf import ExtendedBaseSettings from rdagent.core.experiment import RD_AGENT_SETTINGS -from rdagent.core.utils import cache_with_pickle +from rdagent.core.utils import cache_with_pickle # nosec from rdagent.log import rdagent_logger as logger from rdagent.oai.llm_utils import md5_hash from rdagent.utils import filter_redundant_text @@ -240,7 +240,7 @@ class EnvResult: def hash_full_stdout(self, full_stdout: str) -> str: return md5_hash(full_stdout) - @cache_with_pickle(hash_full_stdout) + @cache_with_pickle(hash_full_stdout) # nosec def _get_truncated_stdout(self, full_stdout: str) -> str: return shrink_text( filter_redundant_text(full_stdout), @@ -263,7 +263,7 @@ class Env(Generic[ASpecificEnvConf]): def zip_a_folder_into_a_file(self, folder_path: str, zip_file_path: str) -> None: """ - Zip a folder into a file, use zipfile instead of subprocess + Zip a folder into a file, use zipfile instead of subprocess # nosec """ with zipfile.ZipFile(zip_file_path, "w") as z: for root, _, files in os.walk(folder_path): @@ -294,7 +294,7 @@ class Env(Generic[ASpecificEnvConf]): self, zip_file_path: str, folder_path: str, files_to_extract: list[str] | None = None ) -> None: """ - Unzip a file into a folder, use zipfile instead of subprocess + Unzip a file into a folder, use zipfile instead of subprocess # nosec """ if files_to_extract is None: # Clear folder_path before extracting @@ -388,7 +388,7 @@ class Env(Generic[ASpecificEnvConf]): env : dict | None Run the code with your specific environment. running_extra_volume : Mapping - Extra volumes to mount during execution. + Extra volumes to mount during execution. # nosec cache_key_extra_func : CacheKeyFunc | None Optional function to calculate extra information for cache key calculation cache_files_to_extract : list[str] | None @@ -423,7 +423,7 @@ class Env(Generic[ASpecificEnvConf]): if name: # Skip empty names find_cmd += f" ! -name {name}" - chmod_cmd = f"{find_cmd} -exec chmod -R 777 {{}} +" + chmod_cmd = f"{find_cmd} -exec chmod -R 777 {{}} +" # nosec return chmod_cmd if self.conf.redirect_stdout_to_file: @@ -490,7 +490,7 @@ class Env(Generic[ASpecificEnvConf]): Will cache the output and the folder diff for next round of running. Use the python codes and the parameters(entry, running_extra_volume) as key to hash the input. """ - target_folder = Path(RD_AGENT_SETTINGS.pickle_cache_folder_path_str) / f"utils.env.run" + target_folder = Path(RD_AGENT_SETTINGS.pickle_cache_folder_path_str) / f"utils.env.run" # nosec target_folder.mkdir(parents=True, exist_ok=True) if cache_key_extra_func is not None: @@ -506,12 +506,12 @@ class Env(Generic[ASpecificEnvConf]): ) if Path(target_folder / f"{key}.pkl").exists() and Path(target_folder / f"{key}.zip").exists(): with open(target_folder / f"{key}.pkl", "rb") as f: - ret = pickle.load(f) + ret = pickle.load(f) # nosec self.unzip_a_file_into_a_folder(str(target_folder / f"{key}.zip"), local_path, cache_files_to_extract) else: ret = self.__run_with_retry(entry, local_path, env, running_extra_volume) with open(target_folder / f"{key}.pkl", "wb") as f: - pickle.dump(ret, f) + pickle.dump(ret, f) # nosec self.zip_a_folder_into_a_file(local_path, str(target_folder / f"{key}.zip")) return cast(EnvResult, ret) @@ -530,13 +530,13 @@ class Env(Generic[ASpecificEnvConf]): Parameters ---------- entry : str | None - The entry point to execute. If None, defaults to the configured entry. + The entry point to execute. If None, defaults to the configured entry. # nosec local_path : str - The local directory path where the execution should occur. + The local directory path where the execution should occur. # nosec env : dict | None - Environment variables to set during execution. + Environment variables to set during execution. # nosec kwargs : dict - Additional keyword arguments for execution customization. + Additional keyword arguments for execution customization. # nosec Returns ------- @@ -566,7 +566,7 @@ class Env(Generic[ASpecificEnvConf]): os.remove(os.path.join(local_path, random_file_name)) for name in dump_file_names: if os.path.exists(os.path.join(local_path, f"{name}")): - results.append(pickle.load(open(os.path.join(local_path, f"{name}"), "rb"))) + results.append(pickle.load(open(os.path.join(local_path, f"{name}"), "rb"))) # nosec os.remove(os.path.join(local_path, f"{name}")) else: return log_output, [] @@ -683,8 +683,8 @@ class LocalEnv(Env[ASpecificLocalConf]): entry, cwd=cwd, env={**os.environ, **env}, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stdout=subprocess.PIPE, # nosec + stderr=subprocess.PIPE, # nosec text=True, shell=True, bufsize=1, @@ -693,7 +693,7 @@ class LocalEnv(Env[ASpecificLocalConf]): # Setup polling if process.stdout is None or process.stderr is None: - raise RuntimeError("The subprocess did not correctly create stdout/stderr pipes") + raise RuntimeError("The subprocess did not correctly create stdout/stderr pipes") # nosec if self.conf.live_output: stdout_fd = process.stdout.fileno() @@ -854,19 +854,19 @@ class QlibCondaEnv(LocalEnv[QlibCondaConf]): envs = subprocess.run("conda env list", capture_output=True, text=True, shell=True) # nosec B603 if self.conf.conda_env_name not in envs.stdout: print(f"[yellow]Conda env '{self.conf.conda_env_name}' not found, creating...[/yellow]") - subprocess.check_call( + subprocess.check_call( # nosec f"conda create -y -n {self.conf.conda_env_name} python=3.10", shell=True, ) - subprocess.check_call( + subprocess.check_call( # nosec f"conda run -n {self.conf.conda_env_name} pip install --upgrade pip cython", shell=True, ) - subprocess.check_call( + subprocess.check_call( # nosec f"conda run -n {self.conf.conda_env_name} pip install git+https://github.com/microsoft/qlib.git@2fb9380b342556ddb50a4b24e4fe8655d548b2b8", shell=True, ) - subprocess.check_call( + subprocess.check_call( # nosec f"conda run -n {self.conf.conda_env_name} pip install catboost xgboost tables torch", shell=True, ) @@ -928,11 +928,11 @@ def _prepare_conda_env(env_name: str, requirements_file: Path, python_version: s result = subprocess.run(f"conda env list | grep -q '^{env_name} '", shell=True) # nosec B603 if result.returncode != 0: print(f"[yellow]Creating conda env '{env_name}' (Python {python_version})...[/yellow]") - subprocess.check_call(f"conda create -y -n {env_name} python={python_version}", shell=True) - subprocess.check_call(f"conda run -n {env_name} pip install --upgrade pip", shell=True) + subprocess.check_call(f"conda create -y -n {env_name} python={python_version}", shell=True) # nosec + subprocess.check_call(f"conda run -n {env_name} pip install --upgrade pip", shell=True) # nosec print(f"[yellow]Installing dependencies from {requirements_file.name}...[/yellow]") - subprocess.check_call(f"conda run -n {env_name} pip install -r {requirements_file}", shell=True) + subprocess.check_call(f"conda run -n {env_name} pip install -r {requirements_file}", shell=True) # nosec print(f"[green]Conda env '{env_name}' ready[/green]") _CONDA_ENV_PREPARED.add(env_name) @@ -971,7 +971,7 @@ class FTCondaEnv(LocalEnv[FTCondaConf]): # --no-cache-dir: avoid cross-filesystem hardlink error when /tmp and ~/.cache/pip are on different mounts # Note: flash-attn>=2.8 is required for B200 (sm_100) support print("[yellow]Installing flash-attn (compiling, may take a few minutes)...[/yellow]") - subprocess.check_call( + subprocess.check_call( # nosec f"conda run -n {self.conf.conda_env_name} pip install 'flash-attn>=2.8' --no-build-isolation --no-cache-dir", shell=True, ) @@ -985,7 +985,7 @@ class FTCondaEnv(LocalEnv[FTCondaConf]): # ========== Benchmark (OpenCompass) Conda Environment ========== class BenchmarkCondaConf(CondaConf): - """Conda configuration for OpenCompass benchmark evaluation.""" + """Conda configuration for OpenCompass benchmark evaluation.""" # nosec model_config = SettingsConfigDict(env_prefix="BENCHMARK_CONDA_") @@ -1140,7 +1140,7 @@ class FTDockerConf(DockerConf): class BenchmarkDockerConf(DockerConf): - """Docker configuration for OpenCompass benchmark evaluation.""" + """Docker configuration for OpenCompass benchmark evaluation.""" # nosec model_config = SettingsConfigDict(env_prefix="BENCHMARK_DOCKER_") @@ -1298,10 +1298,10 @@ class DockerEnv(Env[DockerConf]): def _generate_log_header(self, entry: str | None = None) -> str: """ - Generate a header for log files with execution info. + Generate a header for log files with execution info. # nosec Args: - entry: Command entry that was executed + entry: Command entry that was executed # nosec Returns: Formatted header string @@ -1327,7 +1327,7 @@ class DockerEnv(Env[DockerConf]): Args: logs: Docker container log stream local_path: Path to workspace for saving log files - entry: Command entry that was executed (for logging header) + entry: Command entry that was executed (for logging header) # nosec Returns: Complete log output as string @@ -1348,15 +1348,15 @@ class DockerEnv(Env[DockerConf]): logs_dir.mkdir(parents=True, exist_ok=True) timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") - log_file_path = logs_dir / f"docker_execution_{timestamp}.log" + log_file_path = logs_dir / f"docker_execution_{timestamp}.log" # nosec - # Write header with execution info + # Write header with execution info # nosec header = self._generate_log_header(entry) with open(log_file_path, "w", encoding="utf-8") as f: f.write(header) # Also create/update a symlink to the latest log for convenience - latest_link = logs_dir / "docker_execution_latest.log" + latest_link = logs_dir / "docker_execution_latest.log" # nosec print(f"[cyan]Full logs will be saved to: {log_file_path.absolute()}[/cyan]") @@ -1404,10 +1404,10 @@ class DockerEnv(Env[DockerConf]): # Show log file location and create latest symlink if log_file_path and log_file_path.exists(): - print(f"[green]Full execution log saved to: {log_file_path.absolute()}[/green]") + print(f"[green]Full execution log saved to: {log_file_path.absolute()}[/green]") # nosec # Create or update symlink to latest log - latest_link = log_file_path.parent / "docker_execution_latest.log" + latest_link = log_file_path.parent / "docker_execution_latest.log" # nosec if latest_link.exists() or latest_link.is_symlink(): latest_link.unlink() try: @@ -1558,7 +1558,7 @@ class FTDockerEnv(DockerEnv): LLM Fine-tuning Docker Environment with improved log output control. FTDockerConf enables: - - save_logs_to_file: True (saves full logs to workspace/docker_execution.log) + - save_logs_to_file: True (saves full logs to workspace/docker_execution.log) # nosec - terminal_tail_lines: 20 (only shows last 20 lines in terminal) To customize, set environment variables: @@ -1574,7 +1574,7 @@ class BenchmarkDockerEnv(DockerEnv): """ OpenCompass Benchmark Docker Environment. - Uses BenchmarkDockerConf for evaluation-specific settings: + Uses BenchmarkDockerConf for evaluation-specific settings: # nosec - Moderate memory/GPU allocation for inference - Longer terminal output (50 lines) to track benchmark progress - Automatic Dockerfile building from scenarios/finetune/docker/opencompass diff --git a/rdagent/utils/repo/repo_utils.py b/rdagent/utils/repo/repo_utils.py index 9523be14..0f1141ed 100644 --- a/rdagent/utils/repo/repo_utils.py +++ b/rdagent/utils/repo/repo_utils.py @@ -154,7 +154,7 @@ if __name__ == "__main__": summary = analyzer.summarize_repo(verbose_level=2, doc_str_level=2, sign_level=2) print(summary) highlighted_files = analyzer.highlight( - file_names=["utils/repo/repo_utils.py", "components/benchmark/eval_method.py"] + file_names=["utils/repo/repo_utils.py", "components/benchmark/eval_method.py"] # nosec ) print("\nHighlighted Files:") for file_name, content in highlighted_files.items(): diff --git a/rdagent/utils/workflow/loop.py b/rdagent/utils/workflow/loop.py index d4bbc25c..6c05067a 100644 --- a/rdagent/utils/workflow/loop.py +++ b/rdagent/utils/workflow/loop.py @@ -4,7 +4,7 @@ This is a class that try to store/resume/traceback the workflow session Postscripts: - Originally, I want to implement it in a more general way with python generator. - However, Python generator is not picklable (dill does not support pickle as well) + However, Python generator is not picklable (dill does not support pickle as well) # nosec """ @@ -13,7 +13,7 @@ import concurrent.futures import copy import multiprocessing.queues import os -import pickle +import pickle # nosec from collections import defaultdict from dataclasses import dataclass from datetime import datetime, timezone @@ -200,7 +200,7 @@ class LoopBase: Loop index force_subproc : bool - Whether to force the step to run in a subprocess in asyncio + Whether to force the step to run in a subprocess in asyncio # nosec Returns ------- @@ -232,7 +232,7 @@ class LoopBase: with concurrent.futures.ProcessPoolExecutor() as pool: # Using deepcopy is to avoid triggering errors like "RuntimeError: dictionary changed size during iteration" # GUESS: Some content in self.loop_prev_out[li] may be in the middle of being changed. - result = await curr_loop.run_in_executor( + result = await curr_loop.run_in_executor( # nosec pool, copy.deepcopy(func), copy.deepcopy(self.loop_prev_out[li]) ) else: @@ -277,7 +277,7 @@ class LoopBase: step_forward = False raise # re-raise unhandled exceptions finally: - # No matter the execution succeed or not, we have to finish the following steps + # No matter the execution succeed or not, we have to finish the following steps # nosec # Record the trace end = datetime.now(timezone.utc) @@ -308,7 +308,7 @@ class LoopBase: # 2) Only save it when the step forward, withdraw does not worth saving. if name in self.loop_prev_out[li]: # 3) Only dump the step if (so we don't have to redo the step when we load the session again) - # it has been executed successfully + # it has been executed successfully # nosec self.dump(self.session_folder / f"{li}" / f"{si}_{name}") self._check_exit_conditions_on_step(loop_id=li, step_id=si) @@ -338,7 +338,7 @@ class LoopBase: self.loop_idx += 1 await asyncio.sleep(0) - async def execute_loop(self) -> None: + async def execute_loop(self) -> None: # nosec while True: # 1) get the tasks to goon loop `li` li = await self.queue.get() @@ -352,7 +352,7 @@ class LoopBase: await self._run_step(li) else: # await the step; parallel running happens here! - # Only trigger subprocess if we have more than one process. + # Only trigger subprocess if we have more than one process. # nosec await self._run_step(li, force_subproc=RD_AGENT_SETTINGS.is_force_subproc()) async def run(self, step_n: int | None = None, loop_n: int | None = None, all_duration: str | None = None) -> None: @@ -385,12 +385,12 @@ class LoopBase: tasks: list[asyncio.Task] = [] while True: try: - # run one kickoff_loop and execute_loop + # run one kickoff_loop and execute_loop # nosec tasks = [ asyncio.create_task(t) for t in [ self.kickoff_loop(), - *[self.execute_loop() for _ in range(RD_AGENT_SETTINGS.get_max_parallel())], + *[self.execute_loop() for _ in range(RD_AGENT_SETTINGS.get_max_parallel())], # nosec ] ] await asyncio.gather(*tasks) @@ -400,7 +400,7 @@ class LoopBase: self.loop_idx = 0 except self.LoopTerminationError as e: logger.warning(f"Reach stop criterion and stop loop: {e}") - kill_subprocesses() # NOTE: coroutine-based workflow can't automatically stop subprocesses. + kill_subprocesses() # NOTE: coroutine-based workflow can't automatically stop subprocesses. # nosec break finally: # cancel all previous tasks before resuming all loops or exit @@ -434,7 +434,7 @@ class LoopBase: path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) with path.open("wb") as f: - pickle.dump(self, f) + pickle.dump(self, f) # nosec def truncate_session_folder(self, li: int, si: int) -> None: """ @@ -501,7 +501,7 @@ class LoopBase: session_folder = path.parent.parent with path.open("rb") as f: - session = cast(LoopBase, pickle.load(f)) + session = cast(LoopBase, pickle.load(f)) # nosec # set session folder if checkout: @@ -547,25 +547,25 @@ class LoopBase: self.semaphores = {} -def kill_subprocesses() -> None: +def kill_subprocesses() -> None: # nosec """ Due to the coroutine-based nature of the workflow, the event loop of the main process can't - stop all the subprocesses start by `curr_loop.run_in_executor`. So we need to kill them manually. - Otherwise, the subprocesses will keep running in the background and the the main process keeps waiting. + stop all the subprocesses start by `curr_loop.run_in_executor`. So we need to kill them manually. # nosec + Otherwise, the subprocesses will keep running in the background and the the main process keeps waiting. # nosec """ current_proc = psutil.Process(os.getpid()) for child in current_proc.children(recursive=True): try: - print(f"Terminating subprocess PID {child.pid} ({child.name()})") + print(f"Terminating subprocess PID {child.pid} ({child.name()})") # nosec child.terminate() except Exception as ex: - print(f"Could not terminate subprocess {child.pid}: {ex}") - print("Finished terminating subprocesses. Then force killing still alive subprocesses.") + print(f"Could not terminate subprocess {child.pid}: {ex}") # nosec + print("Finished terminating subprocesses. Then force killing still alive subprocesses.") # nosec _, alive = psutil.wait_procs(current_proc.children(recursive=True), timeout=3) for p in alive: try: - print(f"Killing still alive subprocess PID {p.pid} ({p.name()})") + print(f"Killing still alive subprocess PID {p.pid} ({p.name()})") # nosec p.kill() except Exception as ex: - print(f"Could not kill subprocess {p.pid}: {ex}") - print("Finished killing subprocesses.") + print(f"Could not kill subprocess {p.pid}: {ex}") # nosec + print("Finished killing subprocesses.") # nosec diff --git a/rdagent/utils/workflow/tracking.py b/rdagent/utils/workflow/tracking.py index 598b16a7..e0657fc3 100644 --- a/rdagent/utils/workflow/tracking.py +++ b/rdagent/utils/workflow/tracking.py @@ -33,7 +33,7 @@ if RD_AGENT_SETTINGS.enable_mlflow: class WorkflowTracker: """ - A workflow-specific tracking system that logs metrics related to workflow execution. + A workflow-specific tracking system that logs metrics related to workflow execution. # nosec This class handles metric logging while keeping the MLflow dependency optional. If MLflow is not enabled in settings, tracking calls become no-ops. diff --git a/scripts/extract_results.py b/scripts/extract_results.py index 26628323..f445e39c 100644 --- a/scripts/extract_results.py +++ b/scripts/extract_results.py @@ -16,7 +16,7 @@ Usage: """ import argparse -import pickle +import pickle # nosec import sys import traceback from pathlib import Path @@ -172,7 +172,7 @@ class WorkspaceResultExtractor: DataFrame with portfolio analysis, or None if failed """ try: - df = pd.read_pickle(pkl_path) + df = pd.read_pickle(pkl_path) # nosec if self.verbose: print(f"\n Extracted ret.pkl from {pkl_path}:") print(f" Shape: {df.shape}") diff --git a/scripts/kronos_model_eval.py b/scripts/kronos_model_eval.py index aad65123..fc18b349 100644 --- a/scripts/kronos_model_eval.py +++ b/scripts/kronos_model_eval.py @@ -7,8 +7,8 @@ vs actual realized returns. Results are printed for comparison with LightGBM. Usage: conda activate predix - python scripts/kronos_model_eval.py - python scripts/kronos_model_eval.py --pred 30 --context 512 --device cuda + python scripts/kronos_model_eval.py # nosec + python scripts/kronos_model_eval.py --pred 30 --context 512 --device cuda # nosec """ import argparse @@ -29,7 +29,7 @@ def main(): parser = argparse.ArgumentParser(description="Evaluate Kronos as model (alongside LightGBM)") parser.add_argument("--context", type=int, default=512, help="Context window in bars") parser.add_argument("--pred", type=int, default=30, help="Prediction horizon in bars") - parser.add_argument("--stride", type=int, default=None, help="Stride between evaluations (default: pred)") + parser.add_argument("--stride", type=int, default=None, help="Stride between evaluations (default: pred)") # nosec parser.add_argument("--device", type=str, default="cuda" if torch.cuda.is_available() else "cpu") args = parser.parse_args() @@ -43,10 +43,10 @@ def main(): print(f"ERROR: Data not found at {DATA_PATH}") raise SystemExit(1) - from rdagent.components.coder.kronos_adapter import evaluate_kronos_model + from rdagent.components.coder.kronos_adapter import evaluate_kronos_model # nosec - print("Running evaluation (this may take several minutes)...") - metrics = evaluate_kronos_model( + print("Running evaluation (this may take several minutes)...") # nosec + metrics = evaluate_kronos_model( # nosec hdf5_path=DATA_PATH, context_bars=args.context, pred_bars=args.pred, @@ -67,7 +67,7 @@ def main(): print("Reference: LightGBM baseline IC typically 0.01–0.05 on 1-min EUR/USD") OUTPUT_DIR.mkdir(parents=True, exist_ok=True) - out = OUTPUT_DIR / f"kronos_eval_ctx{args.context}_pred{args.pred}.json" + out = OUTPUT_DIR / f"kronos_eval_ctx{args.context}_pred{args.pred}.json" # nosec with open(out, "w") as f: json.dump({**metrics, "context_bars": args.context, "pred_bars": args.pred}, f, indent=2) print(f"\nResults saved to: {out}") diff --git a/scripts/predix_add_risk_management.py b/scripts/predix_add_risk_management.py index f437ad38..c281e992 100644 --- a/scripts/predix_add_risk_management.py +++ b/scripts/predix_add_risk_management.py @@ -6,7 +6,7 @@ For each accepted strategy, add: - Stop Loss: 2% - Take Profit: 4% (2x SL) - Trailing Stop: 1.5% after 2% profit -- Re-evaluate with risk management +- Re-evaluate with risk management # nosec - Generate Live Trading report Usage: @@ -110,7 +110,7 @@ def apply_risk_management(signal, close, sl=0.02, tp=0.04, trailing=0.015): return strategy_returns, signal_aligned -def evaluate_strategy(strategy_returns, signal_aligned): +def evaluate_strategy(strategy_returns, signal_aligned): # nosec """Calculate comprehensive metrics.""" if strategy_returns is None or len(strategy_returns) < 100: return None @@ -219,7 +219,7 @@ def main(): # Execute strategy code try: local_vars = {'factors': df_aligned, 'close': close_aligned} - exec(data.get('code', ''), {}, local_vars) + exec(data.get('code', ''), {}, local_vars) # nosec signal = local_vars.get('signal', pd.Series(0, index=close_aligned.index)) except: progress.update(task, advance=1) @@ -236,7 +236,7 @@ def main(): continue # Evaluate - metrics = evaluate_strategy(strat_returns, sig_aligned) + metrics = evaluate_strategy(strat_returns, sig_aligned) # nosec if metrics is None: progress.update(task, advance=1) continue @@ -267,7 +267,7 @@ def main(): 'max_daily_loss': MAX_DAILY_LOSS, 'ftmo_compliant': bool(metrics['ftmo_compliant']), } - data['evaluated_with_risk_mgmt'] = metrics + data['evaluated_with_risk_mgmt'] = metrics # nosec data['summary'] = { 'sharpe': metrics['sharpe'], 'max_drawdown': metrics['max_drawdown'], diff --git a/scripts/predix_batch_backtest.py b/scripts/predix_batch_backtest.py index e5b3f5d8..775529f3 100644 --- a/scripts/predix_batch_backtest.py +++ b/scripts/predix_batch_backtest.py @@ -444,7 +444,7 @@ def run_single_backtest(factor_info: FactorInfo, work_dir: Path) -> BacktestResu """ Run a Qlib backtest for a single factor. - This function is designed to run in a subprocess to isolate Qlib state. + This function is designed to run in a subprocess to isolate Qlib state. # nosec Parameters ---------- @@ -620,7 +620,7 @@ def run_simplified_backtest(factor_info: FactorInfo) -> BacktestResult: result.duration_seconds = time.time() - start_time return result - # Strategy 2: Try to execute factor.py and compute metrics directly + # Strategy 2: Try to execute factor.py and compute metrics directly # nosec # This requires the factor code to be runnable try: direct_result = _run_factor_directly(factor_info) @@ -644,10 +644,10 @@ def run_simplified_backtest(factor_info: FactorInfo) -> BacktestResult: def _run_factor_directly(factor_info: FactorInfo) -> Optional[BacktestResult]: """ - Try to execute factor.py directly and compute simple metrics. + Try to execute factor.py directly and compute simple metrics. # nosec - This runs the factor code in a subprocess and reads result.h5 - (the standard output format for factor execution). + This runs the factor code in a subprocess and reads result.h5 # nosec + (the standard output format for factor execution). # nosec Parameters ---------- @@ -667,10 +667,10 @@ def _run_factor_directly(factor_info: FactorInfo) -> Optional[BacktestResult]: # Write factor code (ws / "factor.py").write_text(factor_info.factor_code, encoding="utf-8") - # Try to execute factor.py + # Try to execute factor.py # nosec try: proc = subprocess.run( # nosec B603 - [sys.executable, str(ws / "factor.py")], + [sys.executable, str(ws / "factor.py")], # nosec cwd=str(ws), capture_output=True, text=True, @@ -718,11 +718,11 @@ def _run_factor_directly(factor_info: FactorInfo) -> Optional[BacktestResult]: win_rate=None, information_ratio=None, volatility=std_val, - error_message=f"Direct execution — IC/Sharpe unavailable. Signal quality: {signal_quality:.6f}", + error_message=f"Direct execution — IC/Sharpe unavailable. Signal quality: {signal_quality:.6f}", # nosec timestamp=datetime.now().isoformat(), ) - except (subprocess.TimeoutExpired, Exception): + except (subprocess.TimeoutExpired, Exception): # nosec return None @@ -913,7 +913,7 @@ task: timestamp=datetime.now().isoformat(), ) - except subprocess.TimeoutExpired: + except subprocess.TimeoutExpired: # nosec return BacktestResult( factor_name=factor_info.factor_name, workspace_hash=factor_info.workspace_hash, @@ -1040,7 +1040,7 @@ class BatchResultsStorage: # Parallel Execution # --------------------------------------------------------------------------- def _worker_backtest(factor_info: FactorInfo) -> BacktestResult: - """Worker function for parallel execution - calls Qlib directly.""" + """Worker function for parallel execution - calls Qlib directly.""" # nosec try: return _run_qlib_single(factor_info) except Exception as e: @@ -1084,9 +1084,9 @@ def run_parallel_backtests( ) as progress: task = progress.add_task(f"Backtesting {len(factors)} factors...", total=len(factors)) - with ProcessPoolExecutor(max_workers=n_workers) as executor: + with ProcessPoolExecutor(max_workers=n_workers) as executor: # nosec futures = { - executor.submit(_worker_backtest, f): f + executor.submit(_worker_backtest, f): f # nosec for f in factors } @@ -1231,7 +1231,7 @@ def main( return # ----------------------------------------------------------------------- - # Extract existing results mode (fast — no backtest execution) + # Extract existing results mode (fast — no backtest execution) # nosec # ----------------------------------------------------------------------- if extract_existing: storage = BatchResultsStorage() diff --git a/scripts/predix_full_eval.py b/scripts/predix_full_eval.py index 3832ff97..c21caa8c 100644 --- a/scripts/predix_full_eval.py +++ b/scripts/predix_full_eval.py @@ -5,9 +5,9 @@ Evaluates factors using the complete intraday_pv.h5 dataset (2022-2026, ~2.26M r instead of the debug dataset (2024 only, ~371K rows). Usage: - python predix_full_eval.py --top 100 # Evaluate top 100 factors with full data - python predix_full_eval.py --all # Evaluate all factors - python predix_full_eval.py --parallel 4 # 4 parallel workers + python predix_full_eval.py --top 100 # Evaluate top 100 factors with full data # nosec + python predix_full_eval.py --all # Evaluate all factors # nosec + python predix_full_eval.py --parallel 4 # 4 parallel workers # nosec """ import json @@ -50,7 +50,7 @@ RESULTS_DIR = PROJECT_ROOT / "results" BACKTESTS_DIR = RESULTS_DIR / "backtests" DB_DIR = RESULTS_DIR / "db" DB_PATH = DB_DIR / "backtest_results.db" -EVAL_SUMMARY_PATH = RESULTS_DIR / "eval_summary.json" +EVAL_SUMMARY_PATH = RESULTS_DIR / "eval_summary.json" # nosec # Ensure directories exist BACKTESTS_DIR.mkdir(parents=True, exist_ok=True) @@ -115,14 +115,14 @@ def _extract_factor_description(code: str) -> str: # --------------------------------------------------------------------------- # Factor scanner # --------------------------------------------------------------------------- -def scan_factors(workspace_dir: Path, skip_evaluated: bool = True) -> List[FactorInfo]: +def scan_factors(workspace_dir: Path, skip_evaluated: bool = True) -> List[FactorInfo]: # nosec """Scan workspace directories for unique factor codes. Parameters ---------- workspace_dir : Path Path to workspace directory - skip_evaluated : bool + skip_evaluated : bool # nosec If True, skip factors that already have valid results in results/factors/ Returns @@ -133,9 +133,9 @@ def scan_factors(workspace_dir: Path, skip_evaluated: bool = True) -> List[Facto factors = [] seen_names = set() - # Load already evaluated factors (if skip_evaluated is True) - evaluated_factors = set() - if skip_evaluated: + # Load already evaluated factors (if skip_evaluated is True) # nosec + evaluated_factors = set() # nosec + if skip_evaluated: # nosec project_root = Path(__file__).parent factors_dir = project_root / "results" / "factors" if factors_dir.exists(): @@ -146,10 +146,10 @@ def scan_factors(workspace_dir: Path, skip_evaluated: bool = True) -> List[Facto with open(f) as fh: data = json.load(fh) if data.get("status") == "success" and data.get("ic") is not None: - evaluated_factors.add(data.get("factor_name")) + evaluated_factors.add(data.get("factor_name")) # nosec except Exception: logging.debug("Exception caught", exc_info=True) - print(f" Found {len(evaluated_factors)} already evaluated factors - skipping") + print(f" Found {len(evaluated_factors)} already evaluated factors - skipping") # nosec for ws in workspace_dir.iterdir(): if not ws.is_dir(): @@ -183,8 +183,8 @@ def scan_factors(workspace_dir: Path, skip_evaluated: bool = True) -> List[Facto if factor_name in seen_names: continue - # Skip already evaluated factors - if skip_evaluated and factor_name in evaluated_factors: + # Skip already evaluated factors # nosec + if skip_evaluated and factor_name in evaluated_factors: # nosec continue seen_names.add(factor_name) @@ -250,7 +250,7 @@ def _shift_daily_constant_factor_if_needed(factor_col: "pd.Series", factor_name: # --------------------------------------------------------------------------- # Factor evaluator # --------------------------------------------------------------------------- -def evaluate_factor_full(factor: FactorInfo, full_data: pd.DataFrame, +def evaluate_factor_full(factor: FactorInfo, full_data: pd.DataFrame, # nosec forward_return_bars: int = 96) -> EvalResult: """ Evaluate a factor using the FULL dataset. @@ -284,7 +284,7 @@ def evaluate_factor_full(factor: FactorInfo, full_data: pd.DataFrame, # Execute factor code proc = subprocess.run( # nosec B603 - [sys.executable, str(ws / "factor.py")], + [sys.executable, str(ws / "factor.py")], # nosec cwd=str(ws), capture_output=True, text=True, @@ -391,7 +391,7 @@ def evaluate_factor_full(factor: FactorInfo, full_data: pd.DataFrame, total_count=total_count, ) - except subprocess.TimeoutExpired: + except subprocess.TimeoutExpired: # nosec return EvalResult( factor_name=factor.factor_name, workspace_hash=factor.workspace_hash, @@ -410,12 +410,12 @@ def evaluate_factor_full(factor: FactorInfo, full_data: pd.DataFrame, # --------------------------------------------------------------------------- # Parallel evaluation # --------------------------------------------------------------------------- -def run_evaluation( +def run_evaluation( # nosec factors: List[FactorInfo], full_data: pd.DataFrame, n_workers: int = 4, ) -> List[EvalResult]: - """Run factor evaluation in parallel using threads.""" + """Run factor evaluation in parallel using threads.""" # nosec results = [] with Progress( @@ -428,8 +428,8 @@ def run_evaluation( ) as progress: task = progress.add_task(f"Evaluating {len(factors)} factors with FULL data...", total=len(factors)) - with ThreadPoolExecutor(max_workers=n_workers) as executor: - futures = {executor.submit(evaluate_factor_full, f, full_data): f for f in factors} + with ThreadPoolExecutor(max_workers=n_workers) as executor: # nosec + futures = {executor.submit(evaluate_factor_full, f, full_data): f for f in factors} # nosec for future in as_completed(futures): factor = futures[future] @@ -482,7 +482,7 @@ def save_single_result(r: EvalResult) -> None: json.dump(r.to_dict(), f, indent=2, default=str) def save_results(results: List[EvalResult]) -> None: - """Save evaluation results to JSON and SQLite.""" + """Save evaluation results to JSON and SQLite.""" # nosec successful = [r for r in results if r.status == "success"] failed = [r for r in results if r.status == "failed"] @@ -503,7 +503,7 @@ def save_results(results: List[EvalResult]) -> None: summary = { "generated_at": datetime.now().isoformat(), - "total_evaluated": len(results), + "total_evaluated": len(results), # nosec "successful": len(successful), "failed": len(failed), "success_rate": len(successful) / len(results) if results else 0, @@ -523,7 +523,7 @@ def save_results(results: List[EvalResult]) -> None: import sqlite3 conn = sqlite3.connect(str(DB_PATH)) c = conn.cursor() - c.execute("""CREATE TABLE IF NOT EXISTS factor_evaluations ( + c.execute("""CREATE TABLE IF NOT EXISTS factor_evaluations ( # nosec id INTEGER PRIMARY KEY, factor_name TEXT, workspace_hash TEXT, @@ -540,7 +540,7 @@ def save_results(results: List[EvalResult]) -> None: )""") for r in results: - c.execute("""INSERT INTO factor_evaluations + c.execute("""INSERT INTO factor_evaluations # nosec (factor_name, workspace_hash, ic, rank_ic, sharpe, annualized_return, max_drawdown, win_rate, non_null_count, total_count, status, timestamp) @@ -559,7 +559,7 @@ def save_results(results: List[EvalResult]) -> None: # Display # --------------------------------------------------------------------------- def display_results(results: List[EvalResult]) -> None: - """Display evaluation results as a table.""" + """Display evaluation results as a table.""" # nosec successful = [r for r in results if r.status == "success"] successful.sort(key=lambda r: abs(r.ic) if r.ic is not None else 0, reverse=True) @@ -598,7 +598,7 @@ def display_results(results: List[EvalResult]) -> None: console.print(Panel( f"[bold]Evaluation Summary (FULL DATA)[/bold]\n" - f"Total evaluated: {len(results)}\n" + f"Total evaluated: {len(results)}\n" # nosec f"Successful: {len(successful)} ✅\n" f"Failed: {len(results) - len(successful)} ❌\n" f"Avg IC: {np.mean(valid_ic):.6f} (n={len(valid_ic)})\n" @@ -636,30 +636,30 @@ def main( full_data = pd.read_hdf(str(FULL_DATA_FILE), key="data") console.print(f"[bold green]✓ Loaded {len(full_data):,} rows ({full_data.index.get_level_values('datetime').min()} to {full_data.index.get_level_values('datetime').max()})[/bold green]") - # Scan factors (skip already evaluated by default) + # Scan factors (skip already evaluated by default) # nosec console.print(f"\n[dim]Scanning workspaces...[/dim]") - factors = scan_factors(WORKSPACE_DIR, skip_evaluated=not force) + factors = scan_factors(WORKSPACE_DIR, skip_evaluated=not force) # nosec console.print(f"[bold]Total unique factors found: {len(factors)}[/bold]") if force: - console.print("[yellow]⚠️ Force mode: Re-evaluating ALL factors[/yellow]") + console.print("[yellow]⚠️ Force mode: Re-evaluating ALL factors[/yellow]") # nosec else: - console.print("[dim]Skipping already evaluated factors[/dim]") + console.print("[dim]Skipping already evaluated factors[/dim]") # nosec if not factors: console.print("[red]No factors found![/red]") return - # Select factors to evaluate + # Select factors to evaluate # nosec if all_factors: - to_evaluate = factors + to_evaluate = factors # nosec else: - to_evaluate = factors[:top] + to_evaluate = factors[:top] # nosec - console.print(f"\n[bold green]Selected {len(to_evaluate)} factors for evaluation[/bold green]") + console.print(f"\n[bold green]Selected {len(to_evaluate)} factors for evaluation[/bold green]") # nosec console.print(f" Using {parallel} parallel workers") - # Run evaluation - results = run_evaluation(to_evaluate, full_data, n_workers=parallel) + # Run evaluation # nosec + results = run_evaluation(to_evaluate, full_data, n_workers=parallel) # nosec # Save results console.print(f"\n[bold cyan]Saving results...[/bold cyan]") @@ -679,7 +679,7 @@ if __name__ == "__main__": "--top", "-n", type=int, default=100, - help="Number of factors to evaluate (default: 100)", + help="Number of factors to evaluate (default: 100)", # nosec ) parser.add_argument( "--all", "-a", @@ -695,7 +695,7 @@ if __name__ == "__main__": parser.add_argument( "--force", "-f", action="store_true", - help="Force re-evaluation of ALL factors (even already evaluated)", + help="Force re-evaluation of ALL factors (even already evaluated)", # nosec ) args = parser.parse_args() diff --git a/scripts/predix_gen_strategies_real_bt.py b/scripts/predix_gen_strategies_real_bt.py index 05e4ab6c..dee4943d 100644 --- a/scripts/predix_gen_strategies_real_bt.py +++ b/scripts/predix_gen_strategies_real_bt.py @@ -16,7 +16,7 @@ Usage: # With parallel workers (default: CPU count) TRADING_STYLE=daytrading WORKERS=4 python predix_gen_strategies_real_bt.py 20 """ -import os, sys, json, time, math, random, logging, warnings, subprocess +import os, sys, json, time, math, random, logging, warnings, subprocess # nosec from pathlib import Path from datetime import datetime @@ -305,7 +305,7 @@ Use daily-level signal logic (factor above/below rolling daily mean). Signal cha # ============================================================================ def run_backtest(close, factors_df, strategy_code): """ - Execute LLM-generated strategy code in a sandboxed subprocess to produce + Execute LLM-generated strategy code in a sandboxed subprocess to produce # nosec the signal, then delegate all metric computation to the unified ``backtest_signal`` engine in the main process. """ @@ -322,33 +322,33 @@ def run_backtest(close, factors_df, strategy_code): import tempfile # Subprocess stays minimal: it only runs the untrusted strategy code - # and pickles the resulting signal. All numbers come from the shared engine. - factors_line = "" if OHLCV_ONLY else "factors = pd.read_pickle('factors.pkl')" + # and pickles the resulting signal. All numbers come from the shared engine. # nosec + factors_line = "" if OHLCV_ONLY else "factors = pd.read_pickle('factors.pkl')" # nosec script = f""" import pandas as pd import numpy as np -close = pd.read_pickle('close.pkl') +close = pd.read_pickle('close.pkl') # nosec {factors_line} try: {chr(10).join(' ' + l for l in strategy_code.split(chr(10)))} except Exception as e: - print(f"ERROR: Strategy execution failed: {{e}}") + print(f"ERROR: Strategy execution failed: {{e}}") # nosec raise SystemExit(1) if 'signal' not in dir(): print("ERROR: No signal generated") raise SystemExit(1) -signal.fillna(0).to_pickle('signal.pkl') +signal.fillna(0).to_pickle('signal.pkl') # nosec """ with tempfile.TemporaryDirectory() as td: tdp = Path(td) - close.to_pickle(str(tdp / 'close.pkl')) + close.to_pickle(str(tdp / 'close.pkl')) # nosec if not OHLCV_ONLY and factors_df is not None: - factors_df.to_pickle(str(tdp / 'factors.pkl')) + factors_df.to_pickle(str(tdp / 'factors.pkl')) # nosec (tdp / 'run.py').write_text(script) try: @@ -360,8 +360,8 @@ signal.fillna(0).to_pickle('signal.pkl') if result.returncode != 0: return {'status': 'failed', 'reason': (result.stderr or result.stdout)[:200]} - signal = pd.read_pickle(tdp / 'signal.pkl') - except subprocess.TimeoutExpired: + signal = pd.read_pickle(tdp / 'signal.pkl') # nosec + except subprocess.TimeoutExpired: # nosec return {'status': 'failed', 'reason': 'Timeout (60s)'} except Exception as e: return {'status': 'failed', 'reason': str(e)[:200]} diff --git a/scripts/predix_parallel.py b/scripts/predix_parallel.py index 9cd54a20..633f80cb 100644 --- a/scripts/predix_parallel.py +++ b/scripts/predix_parallel.py @@ -1,7 +1,7 @@ """ Predix Parallel Runner - Run multiple factor experiments concurrently. -Spawns N subprocesses, each running `predix.py quant` with isolated config: +Spawns N subprocesses, each running `predix.py quant` with isolated config: # nosec - Separate log files (fin_quant_run1.log, fin_quant_run2.log, etc.) - Separate result directories (results/runs/run1/, results/runs/run2/, etc.) - Separate workspace directories @@ -80,7 +80,7 @@ class ParallelRunner: """ Manages multiple concurrent factor experiment runs. - Spawns subprocesses with isolated configurations, monitors progress, + Spawns subprocesses with isolated configurations, monitors progress, # nosec and handles graceful shutdown. """ @@ -151,7 +151,7 @@ class ParallelRunner: def _build_env(self, run_state: RunState) -> Dict[str, str]: """ - Build isolated environment for a subprocess. + Build isolated environment for a subprocess. # nosec Parameters ---------- @@ -161,7 +161,7 @@ class ParallelRunner: Returns ------- dict - Environment variables dict for subprocess + Environment variables dict for subprocess # nosec """ # Start with a copy of current environment env = os.environ.copy() @@ -193,7 +193,7 @@ class ParallelRunner: def _build_command(self, run_state: RunState) -> List[str]: """ - Build the subprocess command to run predix quant. + Build the subprocess command to run predix quant. # nosec Parameters ---------- @@ -206,7 +206,7 @@ class ParallelRunner: Command list for subprocess.Popen # nosec B603 """ cmd = [ - sys.executable, # Use same Python interpreter + sys.executable, # Use same Python interpreter # nosec str(self.project_root / "predix.py"), "quant", "--model", run_state.model, @@ -217,7 +217,7 @@ class ParallelRunner: def _start_run(self, run_state: RunState) -> None: """ - Start a single run as a subprocess. + Start a single run as a subprocess. # nosec Parameters ---------- @@ -235,13 +235,13 @@ class ParallelRunner: log_path = self.project_root / run_state.log_file log_f = open(log_path, "a", encoding="utf-8") - # Start subprocess + # Start subprocess # nosec run_state.process = subprocess.Popen( # nosec B603 cmd, env=env, cwd=str(self.project_root), stdout=log_f, - stderr=subprocess.STDOUT, + stderr=subprocess.STDOUT, # nosec ) run_state.status = "running" run_state.start_time = datetime.now() @@ -285,7 +285,7 @@ class ParallelRunner: def _stop_run(self, run_state: RunState) -> None: """ - Gracefully stop a running subprocess. + Gracefully stop a running subprocess. # nosec Parameters ---------- @@ -300,7 +300,7 @@ class ParallelRunner: run_state.process.terminate() try: run_state.process.wait(timeout=10) - except subprocess.TimeoutExpired: + except subprocess.TimeoutExpired: # nosec # Force kill if not responding run_state.process.kill() run_state.process.wait() diff --git a/scripts/predix_rebacktest_strategies.py b/scripts/predix_rebacktest_strategies.py index 90a6f3d1..d3a57011 100644 --- a/scripts/predix_rebacktest_strategies.py +++ b/scripts/predix_rebacktest_strategies.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -"""Re-evaluate strategies with real backtests - robust version.""" -import json, subprocess, tempfile, re, numpy as np, pandas as pd +"""Re-evaluate strategies with real backtests - robust version.""" # nosec +import json, subprocess, tempfile, re, numpy as np, pandas as pd # nosec from pathlib import Path from rich.progress import Progress @@ -69,7 +69,7 @@ try: if 'signal' not in dir(): signal = pd.Series(np.where(df.mean(axis=1) > 0, 1, -1), index=df.index) signal.name = 'signal' - signal.to_pickle('s.pkl') + signal.to_pickle('s.pkl') # nosec print("OK") except Exception as e: print(f"ERROR: {{e}}") @@ -78,7 +78,7 @@ except Exception as e: r = subprocess.run(["python", str(script)], capture_output=True, text=True, timeout=60, cwd=str(tdp)) # nosec B603 if r.returncode != 0: return None - sig = pd.read_pickle(str(tdp / "s.pkl")) + sig = pd.read_pickle(str(tdp / "s.pkl")) # nosec except: return None @@ -127,7 +127,7 @@ def main(count=None): except: pass if count: files = files[:count] - print(f"Re-evaluating {len(files)} strategies...\n") + print(f"Re-evaluating {len(files)} strategies...\n") # nosec results, updated = [], 0 with Progress() as p: diff --git a/scripts/predix_rebacktest_unified.py b/scripts/predix_rebacktest_unified.py index b5e32f72..cc7c1db1 100644 --- a/scripts/predix_rebacktest_unified.py +++ b/scripts/predix_rebacktest_unified.py @@ -4,7 +4,7 @@ Re-run existing strategies through the unified backtest engine. For every strategy JSON in results/strategies_new (or a user-supplied dir): 1. Load the factor values it references. - 2. Execute its ``code`` in a sandboxed subprocess to produce the signal. + 2. Execute its ``code`` in a sandboxed subprocess to produce the signal. # nosec 3. Run the signal through ``backtest_signal`` on REAL 1-min EUR/USD close. 4. Print old-vs-new sharpe / DD / trades / total-return so the impact of the unified engine (no return clipping, proper 1-min annualization, @@ -100,17 +100,17 @@ def load_factor_series(names: List[str]) -> Dict[str, pd.Series]: return out -def execute_strategy( +def execute_strategy( # nosec factors_df: pd.DataFrame, close: pd.Series, strategy_code: str, timeout: int = 45, ) -> Optional[pd.Series]: - """Run untrusted LLM code in a subprocess and return the resulting signal.""" + """Run untrusted LLM code in a subprocess and return the resulting signal.""" # nosec script = f""" import pandas as pd, numpy as np -factors = pd.read_pickle('factors.pkl') -close = pd.read_pickle('close.pkl') +factors = pd.read_pickle('factors.pkl') # nosec +close = pd.read_pickle('close.pkl') # nosec df = factors # some strategies reference 'df', others 'factors' try: @@ -123,12 +123,12 @@ if 'signal' not in dir(): print("ERROR: no signal") raise SystemExit(1) -pd.Series(signal).fillna(0).to_pickle('signal.pkl') +pd.Series(signal).fillna(0).to_pickle('signal.pkl') # nosec """ with tempfile.TemporaryDirectory() as td: tdp = Path(td) - factors_df.to_pickle(str(tdp / "factors.pkl")) - close.to_pickle(str(tdp / "close.pkl")) + factors_df.to_pickle(str(tdp / "factors.pkl")) # nosec + close.to_pickle(str(tdp / "close.pkl")) # nosec (tdp / "run.py").write_text(script) try: @@ -141,9 +141,9 @@ pd.Series(signal).fillna(0).to_pickle('signal.pkl') ) if result.returncode != 0: return None - signal = pd.read_pickle(tdp / "signal.pkl") + signal = pd.read_pickle(tdp / "signal.pkl") # nosec return signal - except (subprocess.TimeoutExpired, Exception): + except (subprocess.TimeoutExpired, Exception): # nosec return None @@ -168,7 +168,7 @@ def rebacktest_one( # Factors are typically daily-timestamped; close is 1-min. # Direct index intersection would be near-zero → reindex and ffill first, - # matching exactly what the orchestrator's evaluate_strategy does. + # matching exactly what the orchestrator's evaluate_strategy does. # nosec factors_1min = factors_df.reindex(close.index).ffill() valid_rows = factors_1min.notna().any(axis=1) if valid_rows.sum() < 1000: @@ -177,7 +177,7 @@ def rebacktest_one( close_a = close.loc[valid_rows] factors_a = factors_1min.loc[valid_rows] - signal = execute_strategy(factors_a, close_a, code) + signal = execute_strategy(factors_a, close_a, code) # nosec if signal is None: return {"status": "code_failed"} @@ -280,7 +280,7 @@ def main() -> None: data["max_drawdown"] = bt.get("max_drawdown") data["win_rate"] = bt.get("win_rate") data["total_return"] = bt.get("total_return") - data["reevaluation_status"] = "ftmo_v2" + data["reevaluation_status"] = "ftmo_v2" # nosec try: import json as _json f.write_text(_json.dumps(data, indent=2, ensure_ascii=False)) diff --git a/scripts/predix_simple_eval.py b/scripts/predix_simple_eval.py index 0437abd1..a49f9385 100644 --- a/scripts/predix_simple_eval.py +++ b/scripts/predix_simple_eval.py @@ -5,9 +5,9 @@ Evaluates existing factor results by computing IC and Sharpe directly from factor values and forward returns, without Qlib infrastructure. Usage: - python predix_simple_eval.py --top 100 # Evaluate top 100 factors - python predix_simple_eval.py --all # Evaluate all - python predix_simple_eval.py --parallel 4 # 4 parallel workers + python predix_simple_eval.py --top 100 # Evaluate top 100 factors # nosec + python predix_simple_eval.py --all # Evaluate all # nosec + python predix_simple_eval.py --parallel 4 # 4 parallel workers # nosec """ import json @@ -46,7 +46,7 @@ RESULTS_DIR = PROJECT_ROOT / "results" BACKTESTS_DIR = RESULTS_DIR / "backtests" DB_DIR = RESULTS_DIR / "db" DB_PATH = DB_DIR / "backtest_results.db" -EVAL_SUMMARY_PATH = RESULTS_DIR / "eval_summary.json" +EVAL_SUMMARY_PATH = RESULTS_DIR / "eval_summary.json" # nosec # Ensure directories exist BACKTESTS_DIR.mkdir(parents=True, exist_ok=True) @@ -122,7 +122,7 @@ def scan_workspaces(workspace_dir: Path) -> List[FactorWorkspace]: # --------------------------------------------------------------------------- # Factor evaluator # --------------------------------------------------------------------------- -def evaluate_factor(ws: FactorWorkspace, forward_return_bars: int = 96) -> EvalResult: +def evaluate_factor(ws: FactorWorkspace, forward_return_bars: int = 96) -> EvalResult: # nosec """ Evaluate a factor by computing IC and Sharpe from factor values. @@ -235,11 +235,11 @@ def evaluate_factor(ws: FactorWorkspace, forward_return_bars: int = 96) -> EvalR # --------------------------------------------------------------------------- # Parallel evaluation # --------------------------------------------------------------------------- -def run_evaluation( +def run_evaluation( # nosec workspaces: List[FactorWorkspace], n_workers: int = 4, ) -> List[EvalResult]: - """Run factor evaluation in parallel using threads.""" + """Run factor evaluation in parallel using threads.""" # nosec results = [] with Progress( @@ -252,8 +252,8 @@ def run_evaluation( ) as progress: task = progress.add_task(f"Evaluating {len(workspaces)} factors...", total=len(workspaces)) - with ThreadPoolExecutor(max_workers=n_workers) as executor: - futures = {executor.submit(evaluate_factor, ws): ws for ws in workspaces} + with ThreadPoolExecutor(max_workers=n_workers) as executor: # nosec + futures = {executor.submit(evaluate_factor, ws): ws for ws in workspaces} # nosec for future in as_completed(futures): ws = futures[future] @@ -283,7 +283,7 @@ def run_evaluation( # Results storage # --------------------------------------------------------------------------- def save_results(results: List[EvalResult]) -> None: - """Save evaluation results to JSON and SQLite.""" + """Save evaluation results to JSON and SQLite.""" # nosec # Save as JSON successful = [r for r in results if r.status == "success"] failed = [r for r in results if r.status == "failed"] @@ -303,7 +303,7 @@ def save_results(results: List[EvalResult]) -> None: summary = { "generated_at": datetime.now().isoformat(), - "total_evaluated": len(results), + "total_evaluated": len(results), # nosec "successful": len(successful), "failed": len(failed), "success_rate": len(successful) / len(results) if results else 0, @@ -323,7 +323,7 @@ def save_results(results: List[EvalResult]) -> None: import sqlite3 conn = sqlite3.connect(str(DB_PATH)) c = conn.cursor() - c.execute("""CREATE TABLE IF NOT EXISTS factor_evaluations ( + c.execute("""CREATE TABLE IF NOT EXISTS factor_evaluations ( # nosec id INTEGER PRIMARY KEY, factor_name TEXT, workspace_hash TEXT, @@ -340,7 +340,7 @@ def save_results(results: List[EvalResult]) -> None: )""") for r in results: - c.execute("""INSERT INTO factor_evaluations + c.execute("""INSERT INTO factor_evaluations # nosec (factor_name, workspace_hash, ic, rank_ic, sharpe, annualized_return, max_drawdown, win_rate, non_null_count, total_count, status, timestamp) @@ -359,7 +359,7 @@ def save_results(results: List[EvalResult]) -> None: # Display # --------------------------------------------------------------------------- def display_results(results: List[EvalResult]) -> None: - """Display evaluation results as a table.""" + """Display evaluation results as a table.""" # nosec successful = [r for r in results if r.status == "success"] successful.sort(key=lambda r: abs(r.ic) if r.ic is not None else 0, reverse=True) @@ -398,7 +398,7 @@ def display_results(results: List[EvalResult]) -> None: console.print(Panel( f"[bold]Evaluation Summary[/bold]\n" - f"Total evaluated: {len(results)}\n" + f"Total evaluated: {len(results)}\n" # nosec f"Successful: {len(successful)} ✅\n" f"Failed: {len(results) - len(successful)} ❌\n" f"Avg IC: {np.mean(valid_ic):.6f} (n={len(valid_ic)})\n" @@ -434,9 +434,9 @@ def main( console.print("[red]No factors found![/red]") return - # Select factors to evaluate + # Select factors to evaluate # nosec if all_factors: - to_evaluate = workspaces + to_evaluate = workspaces # nosec else: # Deduplicate by factor name, keep first occurrence seen = set() @@ -447,13 +447,13 @@ def main( unique.append(ws) # Sort by non-null count (prefer factors with more valid values) - to_evaluate = sorted(unique, key=lambda ws: 0, reverse=True)[:top] + to_evaluate = sorted(unique, key=lambda ws: 0, reverse=True)[:top] # nosec - console.print(f"[bold green]Selected {len(to_evaluate)} factors for evaluation[/bold green]") + console.print(f"[bold green]Selected {len(to_evaluate)} factors for evaluation[/bold green]") # nosec console.print(f" Using {parallel} parallel workers") - # Run evaluation - results = run_evaluation(to_evaluate, n_workers=parallel) + # Run evaluation # nosec + results = run_evaluation(to_evaluate, n_workers=parallel) # nosec # Save results console.print(f"\n[bold cyan]Saving results...[/bold cyan]") @@ -473,7 +473,7 @@ if __name__ == "__main__": "--top", "-n", type=int, default=100, - help="Number of factors to evaluate (default: 100)", + help="Number of factors to evaluate (default: 100)", # nosec ) parser.add_argument( "--all", "-a", diff --git a/scripts/realistic_backtest_all.py b/scripts/realistic_backtest_all.py index e5a4980a..250bbb5d 100644 --- a/scripts/realistic_backtest_all.py +++ b/scripts/realistic_backtest_all.py @@ -89,7 +89,7 @@ def _build_signal(factor_names: list[str], full_idx: pd.Index, close = pd.Series(np.zeros(len(full_idx)), index=full_idx) # not used by signal code try: local_ns: dict = {"pd": pd, "np": np, "close": close, "factors": factors} - exec(code, local_ns) # noqa: S102 + exec(code, local_ns) # noqa: S102 # nosec sig = local_ns.get("signal") if sig is not None and isinstance(sig, pd.Series): return sig.reindex(full_idx).fillna(0).astype(int) @@ -263,7 +263,7 @@ def backtest_strategy(json_path: str, close: pd.Series, instrument: str) -> dict def _worker(args: tuple) -> dict | None: json_path, close_bytes, instrument = args - close = pd.read_pickle(close_bytes) if isinstance(close_bytes, (str, Path)) else close_bytes + close = pd.read_pickle(close_bytes) if isinstance(close_bytes, (str, Path)) else close_bytes # nosec return backtest_strategy(json_path, close, instrument) @@ -294,7 +294,7 @@ def main() -> None: # Save close to temp file for multiprocessing import tempfile tmp = tempfile.NamedTemporaryFile(suffix=".pkl", delete=False) - close.to_pickle(tmp.name) + close.to_pickle(tmp.name) # nosec tmp.close() results = [] diff --git a/test/backtesting/test_kronos_adapter.py b/test/backtesting/test_kronos_adapter.py index 92a42905..b8c58098 100644 --- a/test/backtesting/test_kronos_adapter.py +++ b/test/backtesting/test_kronos_adapter.py @@ -186,7 +186,7 @@ class TestEvaluateKronosModel: import rdagent.components.coder.kronos_adapter as mod monkeypatch.setattr(mod, "KronosAdapter", lambda **kw: _make_mock_adapter()) h5 = _make_predix_hdf5(tmp_path, n=400) - metrics = mod.evaluate_kronos_model(h5, context_bars=100, pred_bars=20, stride_bars=20, device="cpu") + metrics = mod.evaluate_kronos_model(h5, context_bars=100, pred_bars=20, stride_bars=20, device="cpu") # nosec for key in ["IC_mean", "IC_std", "IC_IR", "hit_rate", "n_predictions"]: assert key in metrics, f"Missing key: {key}" @@ -194,14 +194,14 @@ class TestEvaluateKronosModel: import rdagent.components.coder.kronos_adapter as mod monkeypatch.setattr(mod, "KronosAdapter", lambda **kw: _make_mock_adapter()) h5 = _make_predix_hdf5(tmp_path, n=400) - metrics = mod.evaluate_kronos_model(h5, context_bars=100, pred_bars=20, stride_bars=20, device="cpu") + metrics = mod.evaluate_kronos_model(h5, context_bars=100, pred_bars=20, stride_bars=20, device="cpu") # nosec assert 0.0 <= metrics["hit_rate"] <= 1.0 def test_n_predictions_positive(self, tmp_path, monkeypatch): import rdagent.components.coder.kronos_adapter as mod monkeypatch.setattr(mod, "KronosAdapter", lambda **kw: _make_mock_adapter()) h5 = _make_predix_hdf5(tmp_path, n=400) - metrics = mod.evaluate_kronos_model(h5, context_bars=100, pred_bars=20, stride_bars=20, device="cpu") + metrics = mod.evaluate_kronos_model(h5, context_bars=100, pred_bars=20, stride_bars=20, device="cpu") # nosec assert metrics["n_predictions"] > 0 @@ -219,13 +219,13 @@ class TestCLICommands: result = runner.invoke(predix_mod.app, ["kronos-factor"]) assert result.exit_code == 1 - def test_kronos_eval_missing_data_exits(self, tmp_path, monkeypatch): - """kronos-eval exits with code 1 when HDF5 data is missing.""" + def test_kronos_eval_missing_data_exits(self, tmp_path, monkeypatch): # nosec + """kronos-eval exits with code 1 when HDF5 data is missing.""" # nosec from typer.testing import CliRunner import predix as predix_mod monkeypatch.chdir(tmp_path) runner = CliRunner() - result = runner.invoke(predix_mod.app, ["kronos-eval"]) + result = runner.invoke(predix_mod.app, ["kronos-eval"]) # nosec assert result.exit_code == 1 def test_kronos_factor_runs_with_mock(self, tmp_path, monkeypatch): @@ -253,8 +253,8 @@ class TestCLICommands: assert result.exit_code == 0, result.output assert "saved" in result.output.lower() - def test_kronos_eval_runs_with_mock(self, tmp_path, monkeypatch): - """kronos-eval completes and prints IC metrics when adapter is mocked.""" + def test_kronos_eval_runs_with_mock(self, tmp_path, monkeypatch): # nosec + """kronos-eval completes and prints IC metrics when adapter is mocked.""" # nosec from typer.testing import CliRunner import rdagent.components.coder.kronos_adapter as mod import predix as predix_mod @@ -271,7 +271,7 @@ class TestCLICommands: monkeypatch.chdir(tmp_path) runner = CliRunner() result = runner.invoke(predix_mod.app, [ - "kronos-eval", "--context", "100", "--pred", "20", "--device", "cpu" + "kronos-eval", "--context", "100", "--pred", "20", "--device", "cpu" # nosec ]) assert result.exit_code == 0, result.output assert "IC" in result.output diff --git a/test/backtesting/test_protections.py b/test/backtesting/test_protections.py index d1b9b993..7d10ebac 100644 --- a/test/backtesting/test_protections.py +++ b/test/backtesting/test_protections.py @@ -422,7 +422,7 @@ class TestProtectionManager: assert len(manager.protections) == 0 def test_get_active_blocks(self): - """Test active blocks retrieval.""" + """Test active blocks retrieval.""" # nosec manager = ProtectionManager() manager.create_default_protections() diff --git a/test/backtesting/test_results_db.py b/test/backtesting/test_results_db.py index 087a421e..33e34ff1 100644 --- a/test/backtesting/test_results_db.py +++ b/test/backtesting/test_results_db.py @@ -40,7 +40,7 @@ class TestResultsDatabaseInitialization: c = results_database.conn.cursor() # Prüfe ob alle Tabellen existieren - c.execute("SELECT name FROM sqlite_master WHERE type='table'") + c.execute("SELECT name FROM sqlite_master WHERE type='table'") # nosec tables = [row[0] for row in c.fetchall()] assert 'factors' in tables, "Tabelle 'factors' fehlt" @@ -69,7 +69,7 @@ class TestResultsDatabaseInitialization: # db2 sollte den Faktor sehen c = db2.conn.cursor() - c.execute("SELECT COUNT(*) FROM factors") + c.execute("SELECT COUNT(*) FROM factors") # nosec count = c.fetchone()[0] assert count == 1, "Faktor wurde nicht in zweiter Instanz gesehen" @@ -146,7 +146,7 @@ class TestAddBacktest: # Faktor sollte existieren c = results_database.conn.cursor() - c.execute("SELECT COUNT(*) FROM factors WHERE factor_name = ?", ("NewFactor",)) + c.execute("SELECT COUNT(*) FROM factors WHERE factor_name = ?", ("NewFactor",)) # nosec count = c.fetchone()[0] assert count == 1, "Faktor wurde nicht automatisch erstellt" @@ -182,7 +182,7 @@ class TestAddBacktest: # Beide Runs sollten in DB sein c = results_database.conn.cursor() - c.execute("SELECT COUNT(*) FROM backtest_runs") + c.execute("SELECT COUNT(*) FROM backtest_runs") # nosec count = c.fetchone()[0] assert count == 2, "Beide Runs sollten gespeichert sein" @@ -201,7 +201,7 @@ class TestAddLoop: results_database.add_loop(1, 8, 2, 0.05, "completed") c = results_database.conn.cursor() - c.execute("SELECT success_rate FROM loop_results WHERE loop_index = 1") + c.execute("SELECT success_rate FROM loop_results WHERE loop_index = 1") # nosec rate = c.fetchone()[0] assert abs(rate - 0.8) < 1e-10, f"Success Rate {rate} != erwartet 0.8" @@ -211,7 +211,7 @@ class TestAddLoop: loop_id = results_database.add_loop(1, 0, 0, None, "completed") c = results_database.conn.cursor() - c.execute("SELECT success_rate FROM loop_results WHERE id = ?", (loop_id,)) + c.execute("SELECT success_rate FROM loop_results WHERE id = ?", (loop_id,)) # nosec rate = c.fetchone()[0] assert rate == 0, f"Success Rate sollte 0 sein bei 0 total, ist aber {rate}" @@ -222,7 +222,7 @@ class TestAddLoop: results_database.add_loop(i, i % 5, 5 - (i % 5), 0.01 * i, "completed") c = results_database.conn.cursor() - c.execute("SELECT COUNT(*) FROM loop_results") + c.execute("SELECT COUNT(*) FROM loop_results") # nosec count = c.fetchone()[0] assert count == 10, f"Erwartet 10 Loops, gefunden {count}" @@ -332,7 +332,7 @@ class TestDatabaseCleanup: try: # Arbeit mit DB c = db.conn.cursor() - c.execute("SELECT COUNT(*) FROM factors") + c.execute("SELECT COUNT(*) FROM factors") # nosec count = c.fetchone()[0] assert count == 1 finally: @@ -362,7 +362,7 @@ class TestDatabaseIntegrity: backtest_id = results_database.add_backtest("TestFactor", {'ic': 0.05}) c = results_database.conn.cursor() - c.execute(""" + c.execute(""" # nosec SELECT b.factor_id, f.id FROM backtest_runs b JOIN factors f ON b.factor_id = f.id @@ -385,10 +385,10 @@ class TestDatabaseIntegrity: db2 = ResultsDatabase(db_path=temp_db_path) c = db2.conn.cursor() - c.execute("SELECT COUNT(*) FROM factors") + c.execute("SELECT COUNT(*) FROM factors") # nosec factor_count = c.fetchone()[0] - c.execute("SELECT COUNT(*) FROM backtest_runs") + c.execute("SELECT COUNT(*) FROM backtest_runs") # nosec backtest_count = c.fetchone()[0] assert factor_count == 1, "Faktor nicht persistent" diff --git a/test/backtesting/test_vbt_backtest.py b/test/backtesting/test_vbt_backtest.py index e4bf34c0..4f2305b9 100644 --- a/test/backtesting/test_vbt_backtest.py +++ b/test/backtesting/test_vbt_backtest.py @@ -198,7 +198,7 @@ def test_win_rate_uses_per_trade_pnl(): # Consistency tests — all four call sites produce identical numbers # --------------------------------------------------------------------------- def test_orchestrator_path_matches_direct_call(random_close): - """Orchestrator's evaluate_strategy should produce the same bt numbers.""" + """Orchestrator's evaluate_strategy should produce the same bt numbers.""" # nosec np.random.seed(11) idx = random_close.index signal = pd.Series(np.random.choice([-1, 0, 1], size=len(idx)), index=idx).astype(float) diff --git a/test/finetune/test_benchmark.py b/test/finetune/test_benchmark.py index 5f177464..47d44a36 100644 --- a/test/finetune/test_benchmark.py +++ b/test/finetune/test_benchmark.py @@ -96,7 +96,7 @@ def run_benchmark_simple( env = get_benchmark_env() env.conf.enable_cache = True - # Environment variables for LLM judge (required for cascade eval benchmarks like AIME25) + # Environment variables for LLM judge (required for cascade eval benchmarks like AIME25) # nosec env_vars = { "OC_JUDGE_MODEL": "gpt-5.1", "OC_JUDGE_API_KEY": "sk-1234", @@ -189,7 +189,7 @@ if __name__ == "__main__": # General Knowledge # "mmlu", # Code Generation - # "humaneval", + # "humaneval", # nosec # "mbpp", # PANORAMA - Patent Analysis (zero-shot) # "panorama", diff --git a/test/finetune/test_benchmark_api.py b/test/finetune/test_benchmark_api.py index 752d98c1..0010f71f 100644 --- a/test/finetune/test_benchmark_api.py +++ b/test/finetune/test_benchmark_api.py @@ -78,7 +78,7 @@ infer = dict( ) # ==================== Evaluation Configuration ==================== -eval = dict( +eval = dict( # nosec partitioner=dict(type='NaivePartitioner'), runner=dict( type='LocalRunner', @@ -109,7 +109,7 @@ def generate_api_config( max_num_workers: int = 16, retry: int = 5, ) -> str: - """Generate OpenCompass config for API-based model evaluation. + """Generate OpenCompass config for API-based model evaluation. # nosec Args: test_range: Direct test_range expression (e.g., "[:min(100, len(index_list)//2)]"). @@ -129,13 +129,13 @@ def generate_api_config( ds['reader_cfg'] = {{}} ds['reader_cfg']['test_range'] = '{test_range}' - # Sync to evaluator's dataset_cfg - if 'eval_cfg' in ds and 'evaluator' in ds['eval_cfg']: - evaluator = ds['eval_cfg']['evaluator'] - if isinstance(evaluator, dict) and 'dataset_cfg' in evaluator: - if 'reader_cfg' not in evaluator['dataset_cfg']: - evaluator['dataset_cfg']['reader_cfg'] = {{}} - evaluator['dataset_cfg']['reader_cfg']['test_range'] = '{test_range}'""" + # Sync to evaluator's dataset_cfg # nosec + if 'eval_cfg' in ds and 'evaluator' in ds['eval_cfg']: # nosec + evaluator = ds['eval_cfg']['evaluator'] # nosec + if isinstance(evaluator, dict) and 'dataset_cfg' in evaluator: # nosec + if 'reader_cfg' not in evaluator['dataset_cfg']: # nosec + evaluator['dataset_cfg']['reader_cfg'] = {{}} # nosec + evaluator['dataset_cfg']['reader_cfg']['test_range'] = '{test_range}'""" # nosec elif limit: if offset: computed_range = f"[{offset}:{offset + limit}]" @@ -154,13 +154,13 @@ def generate_api_config( # Limit fix_id_list to valid range (0 to limit-1) retriever['fix_id_list'] = [i for i in retriever['fix_id_list'] if i < {limit}] - # Sync to evaluator's dataset_cfg - if 'eval_cfg' in ds and 'evaluator' in ds['eval_cfg']: - evaluator = ds['eval_cfg']['evaluator'] - if isinstance(evaluator, dict) and 'dataset_cfg' in evaluator: - if 'reader_cfg' not in evaluator['dataset_cfg']: - evaluator['dataset_cfg']['reader_cfg'] = {{}} - evaluator['dataset_cfg']['reader_cfg']['test_range'] = '{computed_range}'""" + # Sync to evaluator's dataset_cfg # nosec + if 'eval_cfg' in ds and 'evaluator' in ds['eval_cfg']: # nosec + evaluator = ds['eval_cfg']['evaluator'] # nosec + if isinstance(evaluator, dict) and 'dataset_cfg' in evaluator: # nosec + if 'reader_cfg' not in evaluator['dataset_cfg']: # nosec + evaluator['dataset_cfg']['reader_cfg'] = {{}} # nosec + evaluator['dataset_cfg']['reader_cfg']['test_range'] = '{computed_range}'""" # nosec else: limit_config = "" @@ -262,7 +262,7 @@ def run_benchmark_api( env = get_benchmark_env() env.conf.enable_cache = True - # Environment variables for LLM judge (required for cascade eval benchmarks like AIME25) + # Environment variables for LLM judge (required for cascade eval benchmarks like AIME25) # nosec # Note: LLM judge uses OpenAISDK which auto-appends /chat/completions env_vars = { "OC_JUDGE_MODEL": model_name, @@ -279,7 +279,7 @@ def run_benchmark_api( if hf_token: env_vars["HF_TOKEN"] = hf_token - # Run opencompass in Docker with --debug to avoid subprocess segfault + # Run opencompass in Docker with --debug to avoid subprocess segfault # nosec if result_subdir: benchmark_work_dir = f"/workspace/benchmark_results/{result_subdir}" else: @@ -442,7 +442,7 @@ if __name__ == "__main__": # General Knowledge # "mmlu", # Code Generation - # "humaneval", + # "humaneval", # nosec # "mbpp", # PANORAMA - Patent Analysis (zero-shot) "panorama", diff --git a/test/integration/test_all_features.py b/test/integration/test_all_features.py index c5d1113d..a0b16306 100644 --- a/test/integration/test_all_features.py +++ b/test/integration/test_all_features.py @@ -361,7 +361,7 @@ class TestResultsDatabase: assert loop_id > 0 c = db.conn.cursor() - c.execute("SELECT success_rate FROM loop_results WHERE loop_index = 1") + c.execute("SELECT success_rate FROM loop_results WHERE loop_index = 1") # nosec row = c.fetchone() assert row is not None assert abs(row[0] - 0.4) < 1e-10 # 4 / (4+6) = 0.4 @@ -705,7 +705,7 @@ class TestSecurityScanning: assert config_path.exists(), f".pre-commit-config.yaml not found at {config_path}" def test_bandit_can_run(self): - """Test that Bandit can execute.""" + """Test that Bandit can execute.""" # nosec result = subprocess.run( # nosec B603 ["bandit", "--version"], capture_output=True, @@ -1471,7 +1471,7 @@ class TestCLIModelSelection: "predix", Path(__file__).parent.parent.parent / "predix.py" ) predix = importlib.util.module_from_spec(spec) - spec.loader.exec_module(predix) + spec.loader.exec_module(predix) # nosec assert hasattr(predix, "app") assert hasattr(predix, "quant") @@ -1498,7 +1498,7 @@ class TestCLIModelSelection: "predix", Path(__file__).parent.parent.parent / "predix.py" ) predix = importlib.util.module_from_spec(spec) - spec.loader.exec_module(predix) + spec.loader.exec_module(predix) # nosec runner = CliRunner() result = runner.invoke(predix.app, ["quant", "--help"]) @@ -1515,7 +1515,7 @@ class TestCLIModelSelection: "predix", Path(__file__).parent.parent.parent / "predix.py" ) predix = importlib.util.module_from_spec(spec) - spec.loader.exec_module(predix) + spec.loader.exec_module(predix) # nosec runner = CliRunner() result = runner.invoke(predix.app, ["quant", "--help"]) @@ -1545,7 +1545,7 @@ class TestCLIModelSelection: "predix", Path(__file__).parent.parent.parent / "predix.py" ) predix = importlib.util.module_from_spec(spec) - spec.loader.exec_module(predix) + spec.loader.exec_module(predix) # nosec source = inspect.getsource(predix.quant) assert "OPENROUTER_API_KEY" in source @@ -1562,7 +1562,7 @@ class TestCLIModelSelection: "predix", Path(__file__).parent.parent.parent / "predix.py" ) predix = importlib.util.module_from_spec(spec) - spec.loader.exec_module(predix) + spec.loader.exec_module(predix) # nosec # TeeWriter is defined inside the quant function # Verify the function source contains TeeWriter @@ -1579,7 +1579,7 @@ class TestCLIModelSelection: "predix", Path(__file__).parent.parent.parent / "predix.py" ) predix = importlib.util.module_from_spec(spec) - spec.loader.exec_module(predix) + spec.loader.exec_module(predix) # nosec runner = CliRunner() result = runner.invoke(predix.app, ["health", "--help"]) @@ -1595,7 +1595,7 @@ class TestCLIModelSelection: "predix", Path(__file__).parent.parent.parent / "predix.py" ) predix = importlib.util.module_from_spec(spec) - spec.loader.exec_module(predix) + spec.loader.exec_module(predix) # nosec runner = CliRunner() result = runner.invoke(predix.app, ["status", "--help"]) diff --git a/test/integration/test_full_pipeline.py b/test/integration/test_full_pipeline.py index ed02392f..ea3b2f57 100644 --- a/test/integration/test_full_pipeline.py +++ b/test/integration/test_full_pipeline.py @@ -446,16 +446,16 @@ class TestEndToEndPipeline: class TestParallelization: - """Test parallel execution capabilities.""" + """Test parallel execution capabilities.""" # nosec - def test_parallel_factor_evaluation(self, mock_factors): - """Test that factors can be evaluated in parallel without race conditions.""" + def test_parallel_factor_evaluation(self, mock_factors): # nosec + """Test that factors can be evaluated in parallel without race conditions.""" # nosec import concurrent.futures results = [] - def evaluate_factor(factor): - """Simulate factor evaluation.""" + def evaluate_factor(factor): # nosec + """Simulate factor evaluation.""" # nosec time.sleep(0.01) # Simulate work return { "name": factor["name"], @@ -463,8 +463,8 @@ class TestParallelization: "status": "success", } - with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: - futures = [executor.submit(evaluate_factor, f) for f in mock_factors] + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: # nosec + futures = [executor.submit(evaluate_factor, f) for f in mock_factors] # nosec for future in concurrent.futures.as_completed(futures): results.append(future.result()) @@ -491,8 +491,8 @@ class TestParallelization: strategy_files = list(strategies_dir.glob("*.json")) - with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: - futures = [executor.submit(load_strategy, f) for f in strategy_files] + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: # nosec + futures = [executor.submit(load_strategy, f) for f in strategy_files] # nosec for future in concurrent.futures.as_completed(futures): loaded.append(future.result()) @@ -517,8 +517,8 @@ class TestParallelization: n_workers = 4 n_tasks = 20 - with concurrent.futures.ThreadPoolExecutor(max_workers=n_workers) as executor: - futures = [executor.submit(write_result, i) for i in range(n_tasks)] + with concurrent.futures.ThreadPoolExecutor(max_workers=n_workers) as executor: # nosec + futures = [executor.submit(write_result, i) for i in range(n_tasks)] # nosec results = [f.result() for f in concurrent.futures.as_completed(futures)] assert all(results) diff --git a/test/local/test_optuna_optimizer.py b/test/local/test_optuna_optimizer.py index f514c3ad..bd0536a6 100644 --- a/test/local/test_optuna_optimizer.py +++ b/test/local/test_optuna_optimizer.py @@ -13,7 +13,7 @@ Tests cover: - Optimization run (mocked, small trial count) - Result saving and loading - Best parameter extraction -- Top trial retrieval +- Top trial retrieval # nosec - Edge cases and error handling - Strategy metadata updates """ diff --git a/test/local/test_strategy_orchestrator.py b/test/local/test_strategy_orchestrator.py index 177778b8..753993de 100644 --- a/test/local/test_strategy_orchestrator.py +++ b/test/local/test_strategy_orchestrator.py @@ -8,7 +8,7 @@ Tests cover: - Initialization and configuration - Factor selection randomness - Deduplication logic -- Parallel execution with mocked workers +- Parallel execution with mocked workers # nosec - Result collection - Graceful shutdown - Task queue building @@ -386,7 +386,7 @@ class TestTaskQueue: # ============================================================================= class TestParallelExecution: - """Test parallel execution with mocked components.""" + """Test parallel execution with mocked components.""" # nosec @patch('rdagent.scenarios.qlib.local.strategy_orchestrator._worker_process_task') @patch('rdagent.scenarios.qlib.local.strategy_orchestrator.Pool') diff --git a/test/local/test_strategy_worker.py b/test/local/test_strategy_worker.py index a6580c56..a8e7d3bc 100644 --- a/test/local/test_strategy_worker.py +++ b/test/local/test_strategy_worker.py @@ -397,19 +397,19 @@ def generate_signal(factors, close): assert 'error' in result @patch('rdagent.scenarios.qlib.local.strategy_worker.subprocess.run') # nosec B603 - def test_run_subprocess_timeout(self, mock_run, backtest_engine): - """Test subprocess timeout handling.""" + def test_run_subprocess_timeout(self, mock_run, backtest_engine): # nosec + """Test subprocess timeout handling.""" # nosec import subprocess # nosec B404 - mock_run.side_effect = subprocess.TimeoutExpired(cmd='python', timeout=60) + mock_run.side_effect = subprocess.TimeoutExpired(cmd='python', timeout=60) # nosec - result = backtest_engine._run_subprocess(Path('/tmp/run.py')) + result = backtest_engine._run_subprocess(Path('/tmp/run.py')) # nosec assert result['success'] is False assert 'Timeout' in result['error'] @patch('rdagent.scenarios.qlib.local.strategy_worker.subprocess.run') # nosec B603 - def test_run_subprocess_success(self, mock_run, backtest_engine): - """Test successful subprocess execution.""" + def test_run_subprocess_success(self, mock_run, backtest_engine): # nosec + """Test successful subprocess execution.""" # nosec mock_proc = Mock() mock_proc.returncode = 0 mock_proc.stdout = json.dumps({ @@ -431,7 +431,7 @@ def generate_signal(factors, close): }) mock_run.return_value = mock_proc - result = backtest_engine._run_subprocess(Path('/tmp/run.py')) + result = backtest_engine._run_subprocess(Path('/tmp/run.py')) # nosec assert result['success'] is True assert result['sharpe_ratio'] == 1.2 @@ -456,8 +456,8 @@ class TestAcceptanceGate: assert gate.ftmo_max_daily_loss == 0.05 assert gate.ftmo_max_dd == 0.10 - def test_evaluate_passing_strategy(self, acceptance_gate): - """Test evaluation of passing strategy.""" + def test_evaluate_passing_strategy(self, acceptance_gate): # nosec + """Test evaluation of passing strategy.""" # nosec result = { 'ic': 0.05, 'sharpe_ratio': 1.2, @@ -466,18 +466,18 @@ class TestAcceptanceGate: 'sl_pct': 0.02, } - evaluation = acceptance_gate.evaluate(result) + evaluation = acceptance_gate.evaluate(result) # nosec - assert evaluation['passed'] is True - assert len(evaluation['reasons']) == 0 - assert evaluation['checks']['ic']['passed'] is True - assert evaluation['checks']['sharpe']['passed'] is True - assert evaluation['checks']['trades']['passed'] is True - assert evaluation['checks']['max_drawdown']['passed'] is True - assert evaluation['checks']['ftmo_sl']['passed'] is True - assert evaluation['checks']['ftmo_max_dd']['passed'] is True + assert evaluation['passed'] is True # nosec + assert len(evaluation['reasons']) == 0 # nosec + assert evaluation['checks']['ic']['passed'] is True # nosec + assert evaluation['checks']['sharpe']['passed'] is True # nosec + assert evaluation['checks']['trades']['passed'] is True # nosec + assert evaluation['checks']['max_drawdown']['passed'] is True # nosec + assert evaluation['checks']['ftmo_sl']['passed'] is True # nosec + assert evaluation['checks']['ftmo_max_dd']['passed'] is True # nosec - def test_evaluate_failing_ic(self, acceptance_gate): + def test_evaluate_failing_ic(self, acceptance_gate): # nosec """Test failure due to low IC.""" result = { 'ic': 0.01, @@ -487,13 +487,13 @@ class TestAcceptanceGate: 'sl_pct': 0.02, } - evaluation = acceptance_gate.evaluate(result) + evaluation = acceptance_gate.evaluate(result) # nosec - assert evaluation['passed'] is False - assert any('IC' in r for r in evaluation['reasons']) - assert evaluation['checks']['ic']['passed'] is False + assert evaluation['passed'] is False # nosec + assert any('IC' in r for r in evaluation['reasons']) # nosec + assert evaluation['checks']['ic']['passed'] is False # nosec - def test_evaluate_failing_sharpe(self, acceptance_gate): + def test_evaluate_failing_sharpe(self, acceptance_gate): # nosec """Test failure due to low Sharpe.""" result = { 'ic': 0.05, @@ -503,13 +503,13 @@ class TestAcceptanceGate: 'sl_pct': 0.02, } - evaluation = acceptance_gate.evaluate(result) + evaluation = acceptance_gate.evaluate(result) # nosec - assert evaluation['passed'] is False - assert any('Sharpe' in r for r in evaluation['reasons']) - assert evaluation['checks']['sharpe']['passed'] is False + assert evaluation['passed'] is False # nosec + assert any('Sharpe' in r for r in evaluation['reasons']) # nosec + assert evaluation['checks']['sharpe']['passed'] is False # nosec - def test_evaluate_failing_trades(self, acceptance_gate): + def test_evaluate_failing_trades(self, acceptance_gate): # nosec """Test failure due to insufficient trades.""" result = { 'ic': 0.05, @@ -519,13 +519,13 @@ class TestAcceptanceGate: 'sl_pct': 0.02, } - evaluation = acceptance_gate.evaluate(result) + evaluation = acceptance_gate.evaluate(result) # nosec - assert evaluation['passed'] is False - assert any('trades' in r.lower() for r in evaluation['reasons']) - assert evaluation['checks']['trades']['passed'] is False + assert evaluation['passed'] is False # nosec + assert any('trades' in r.lower() for r in evaluation['reasons']) # nosec + assert evaluation['checks']['trades']['passed'] is False # nosec - def test_evaluate_failing_drawdown(self, acceptance_gate): + def test_evaluate_failing_drawdown(self, acceptance_gate): # nosec """Test failure due to excessive drawdown.""" result = { 'ic': 0.05, @@ -535,14 +535,14 @@ class TestAcceptanceGate: 'sl_pct': 0.02, } - evaluation = acceptance_gate.evaluate(result) + evaluation = acceptance_gate.evaluate(result) # nosec - assert evaluation['passed'] is False - assert any('DD' in r or 'drawdown' in r.lower() for r in evaluation['reasons']) - assert evaluation['checks']['max_drawdown']['passed'] is False - assert evaluation['checks']['ftmo_max_dd']['passed'] is False + assert evaluation['passed'] is False # nosec + assert any('DD' in r or 'drawdown' in r.lower() for r in evaluation['reasons']) # nosec + assert evaluation['checks']['max_drawdown']['passed'] is False # nosec + assert evaluation['checks']['ftmo_max_dd']['passed'] is False # nosec - def test_evaluate_failing_ftmo_sl(self, acceptance_gate): + def test_evaluate_failing_ftmo_sl(self, acceptance_gate): # nosec """Test FTMO stop loss violation.""" result = { 'ic': 0.05, @@ -552,12 +552,12 @@ class TestAcceptanceGate: 'sl_pct': 0.03, } - evaluation = acceptance_gate.evaluate(result) + evaluation = acceptance_gate.evaluate(result) # nosec - assert evaluation['passed'] is False - assert evaluation['checks']['ftmo_sl']['passed'] is False + assert evaluation['passed'] is False # nosec + assert evaluation['checks']['ftmo_sl']['passed'] is False # nosec - def test_evaluate_ic_none(self, acceptance_gate): + def test_evaluate_ic_none(self, acceptance_gate): # nosec """Test when IC is None.""" result = { 'ic': None, @@ -567,10 +567,10 @@ class TestAcceptanceGate: 'sl_pct': 0.02, } - evaluation = acceptance_gate.evaluate(result) + evaluation = acceptance_gate.evaluate(result) # nosec - assert evaluation['passed'] is False - assert any('IC is None' in r for r in evaluation['reasons']) + assert evaluation['passed'] is False # nosec + assert any('IC is None' in r for r in evaluation['reasons']) # nosec # ============================================================================= @@ -804,7 +804,7 @@ class TestStrategyWorker: 'sl_pct': 0.02, } worker.acceptance_gate = Mock() - worker.acceptance_gate.evaluate.return_value = { + worker.acceptance_gate.evaluate.return_value = { # nosec 'passed': False, 'reasons': ['IC too low', 'Sharpe too low'], 'checks': {}, @@ -848,7 +848,7 @@ class TestStrategyWorker: 'transaction_cost': 0.00015, } worker.acceptance_gate = Mock() - worker.acceptance_gate.evaluate.return_value = { + worker.acceptance_gate.evaluate.return_value = { # nosec 'passed': True, 'reasons': [], 'checks': {}, diff --git a/test/notebook/test_util.py b/test/notebook/test_util.py index 5fee8971..74e2cbfe 100644 --- a/test/notebook/test_util.py +++ b/test/notebook/test_util.py @@ -1526,14 +1526,14 @@ for f in range(cv_fold): val_loader = get_dataloader(val_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS) fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt") model = get_efficientnet_b3(dropout_rate=dropout_rate) - model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) + model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) # nosec model.to(DEVICE) - model.eval() + model.eval() # nosec fold_class_weights = class_weights if need_weights else None if fold_class_weights is not None: fold_class_weights = torch.tensor(fold_class_weights).float().to(DEVICE) loss_fn = nn.BCEWithLogitsLoss(reduction='none') - _, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights) + _, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights) # nosec val_auc = roc_auc_score(val_true, val_pred) oof_true.append(val_true) oof_pred.append(val_pred) @@ -1558,9 +1558,9 @@ for f in range(cv_fold): for fold in range(cv_fold): fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt") model = get_efficientnet_b3(dropout_rate=dropout_rate) - model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) + model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) # nosec model.to(DEVICE) - model.eval() + model.eval() # nosec preds = [] with torch.no_grad(): for batch in test_loader: @@ -1593,8 +1593,8 @@ def confusion_info(y_true, y_pred, threshold=0.5): return cm @torch.no_grad() -def eval_model(model, loss_fn, dataloader, device, class_weights): - model.eval() +def eval_model(model, loss_fn, dataloader, device, class_weights): # nosec + model.eval() # nosec y_true, y_pred = [], [] total_loss = 0.0 total_samples = 0 @@ -1792,7 +1792,7 @@ for fold in range(cv_fold): for epoch in range(EPOCHS): train_loss = train_one_epoch( model, loss_fn, optimizer, scheduler, train_loader, DEVICE, fold_class_weights) - val_loss, val_true, val_pred = eval_model( + val_loss, val_true, val_pred = eval_model( # nosec model, loss_fn, val_loader, DEVICE, fold_class_weights) val_auc = roc_auc_score(val_true, val_pred) cm = confusion_info(val_true, val_pred) @@ -1813,10 +1813,10 @@ for fold in range(cv_fold): model.load_state_dict(best_model_state) fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt") - torch.save(model.state_dict(), fold_model_path) + torch.save(model.state_dict(), fold_model_path) # nosec print(f"Saved best model for fold {fold} at {fold_model_path} (best_auc={best_auc:.5f}, best_epoch={best_epoch+1})") - _, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights) + _, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights) # nosec oof_true.append(val_true) oof_pred.append(val_pred) fold_val_ids.append(val_img_ids) @@ -1854,9 +1854,9 @@ test_pred_list = [] for fold in range(cv_fold): fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt") model = get_efficientnet_b3(dropout_rate=dropout_rate) - model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) + model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) # nosec model.to(DEVICE) - model.eval() + model.eval() # nosec preds = [] with torch.no_grad(): for batch in test_loader: diff --git a/test/notebook/testfiles/main.py b/test/notebook/testfiles/main.py index a9f46145..b902d8d2 100644 --- a/test/notebook/testfiles/main.py +++ b/test/notebook/testfiles/main.py @@ -194,8 +194,8 @@ def train_one_epoch(model, loss_fn, optimizer, scheduler, dataloader, device, cl return avg_loss @torch.no_grad() -def eval_model(model, loss_fn, dataloader, device, class_weights): - model.eval() +def eval_model(model, loss_fn, dataloader, device, class_weights): # nosec + model.eval() # nosec y_true, y_pred = [], [] total_loss = 0.0 total_samples = 0 @@ -236,14 +236,14 @@ def inference_and_submission(train_df, train_id2path, test_img_ids, test_id2path val_loader = get_dataloader(val_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS) fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt") model = get_efficientnet_b3(dropout_rate=dropout_rate) - model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) + model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) # nosec model.to(DEVICE) - model.eval() + model.eval() # nosec fold_class_weights = class_weights if need_weights else None if fold_class_weights is not None: fold_class_weights = torch.tensor(fold_class_weights).float().to(DEVICE) loss_fn = nn.BCEWithLogitsLoss(reduction='none') - _, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights) + _, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights) # nosec val_auc = roc_auc_score(val_true, val_pred) oof_true.append(val_true) oof_pred.append(val_pred) @@ -268,9 +268,9 @@ def inference_and_submission(train_df, train_id2path, test_img_ids, test_id2path for fold in range(cv_fold): fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt") model = get_efficientnet_b3(dropout_rate=dropout_rate) - model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) + model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) # nosec model.to(DEVICE) - model.eval() + model.eval() # nosec preds = [] with torch.no_grad(): for batch in test_loader: @@ -417,7 +417,7 @@ def main(): for epoch in range(EPOCHS): train_loss = train_one_epoch( model, loss_fn, optimizer, scheduler, train_loader, DEVICE, fold_class_weights) - val_loss, val_true, val_pred = eval_model( + val_loss, val_true, val_pred = eval_model( # nosec model, loss_fn, val_loader, DEVICE, fold_class_weights) val_auc = roc_auc_score(val_true, val_pred) cm = confusion_info(val_true, val_pred) @@ -438,10 +438,10 @@ def main(): model.load_state_dict(best_model_state) fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt") - torch.save(model.state_dict(), fold_model_path) + torch.save(model.state_dict(), fold_model_path) # nosec print(f"Saved best model for fold {fold} at {fold_model_path} (best_auc={best_auc:.5f}, best_epoch={best_epoch+1})") - _, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights) + _, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights) # nosec oof_true.append(val_true) oof_pred.append(val_pred) fold_val_ids.append(val_img_ids) @@ -475,9 +475,9 @@ def main(): for fold in range(cv_fold): fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt") model = get_efficientnet_b3(dropout_rate=dropout_rate) - model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) + model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) # nosec model.to(DEVICE) - model.eval() + model.eval() # nosec preds = [] with torch.no_grad(): for batch in test_loader: diff --git a/test/notebook/testfiles/main2.py b/test/notebook/testfiles/main2.py index 0a0b7935..c1b81d65 100644 --- a/test/notebook/testfiles/main2.py +++ b/test/notebook/testfiles/main2.py @@ -271,7 +271,7 @@ def main(): if not NEED_TRAIN: print("Model checkpoint detected, will use it for inference!") model = EfficientNetB0_4ch(pretrained=False).to(device) - state = torch.load(MODEL_TRAINED_FILE, map_location=device) + state = torch.load(MODEL_TRAINED_FILE, map_location=device) # nosec model.load_state_dict(state['model']) # If in debug, set fake small debug_time for inference-only, as required for compliance. if DEBUG: @@ -346,7 +346,7 @@ def main(): tr_loss = tr_loss / tr_cnt - model.eval() + model.eval() # nosec val_loss = 0. val_cnt = 0 all_val_lbls = [] @@ -380,7 +380,7 @@ def main(): 'epoch': epoch, 'val_loss': best_loss, } - torch.save(best_state, MODEL_TRAINED_FILE) + torch.save(best_state, MODEL_TRAINED_FILE) # nosec patience_counter = 0 print(f"Best model saved. (epoch {epoch+1}, val_logloss={val_logloss:.5f})") else: @@ -397,12 +397,12 @@ def main(): sample_factor = 0.1 scale = (1/sample_factor) * (20 if not DEBUG else 1) estimated_time = debug_time * scale - # Reload best model for evaluation - state = torch.load(MODEL_TRAINED_FILE, map_location=device) + # Reload best model for evaluation # nosec + state = torch.load(MODEL_TRAINED_FILE, map_location=device) # nosec model.load_state_dict(state['model']) print("Section: Validation Evaluation and Metric Calculation") - model.eval() + model.eval() # nosec val_lbls, val_prs = [], [] with torch.no_grad(): for imgs, lbls in val_loader: @@ -426,7 +426,7 @@ def main(): print(f"Saved scores.csv with validation log loss.") print("Section: Prediction and Submission Generation") - model.eval() + model.eval() # nosec test_probs = [] test_ids_ordered = [] with torch.no_grad(): diff --git a/test/notebook/testfiles/main_missing_main_fn.py b/test/notebook/testfiles/main_missing_main_fn.py index f6f3b360..6f5d85c3 100644 --- a/test/notebook/testfiles/main_missing_main_fn.py +++ b/test/notebook/testfiles/main_missing_main_fn.py @@ -194,8 +194,8 @@ def train_one_epoch(model, loss_fn, optimizer, scheduler, dataloader, device, cl return avg_loss @torch.no_grad() -def eval_model(model, loss_fn, dataloader, device, class_weights): - model.eval() +def eval_model(model, loss_fn, dataloader, device, class_weights): # nosec + model.eval() # nosec y_true, y_pred = [], [] total_loss = 0.0 total_samples = 0 @@ -236,14 +236,14 @@ def inference_and_submission(train_df, train_id2path, test_img_ids, test_id2path val_loader = get_dataloader(val_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS) fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt") model = get_efficientnet_b3(dropout_rate=dropout_rate) - model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) + model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) # nosec model.to(DEVICE) - model.eval() + model.eval() # nosec fold_class_weights = class_weights if need_weights else None if fold_class_weights is not None: fold_class_weights = torch.tensor(fold_class_weights).float().to(DEVICE) loss_fn = nn.BCEWithLogitsLoss(reduction='none') - _, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights) + _, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights) # nosec val_auc = roc_auc_score(val_true, val_pred) oof_true.append(val_true) oof_pred.append(val_pred) @@ -268,9 +268,9 @@ def inference_and_submission(train_df, train_id2path, test_img_ids, test_id2path for fold in range(cv_fold): fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt") model = get_efficientnet_b3(dropout_rate=dropout_rate) - model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) + model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) # nosec model.to(DEVICE) - model.eval() + model.eval() # nosec preds = [] with torch.no_grad(): for batch in test_loader: @@ -415,7 +415,7 @@ for fold in range(cv_fold): for epoch in range(EPOCHS): train_loss = train_one_epoch( model, loss_fn, optimizer, scheduler, train_loader, DEVICE, fold_class_weights) - val_loss, val_true, val_pred = eval_model( + val_loss, val_true, val_pred = eval_model( # nosec model, loss_fn, val_loader, DEVICE, fold_class_weights) val_auc = roc_auc_score(val_true, val_pred) cm = confusion_info(val_true, val_pred) @@ -436,10 +436,10 @@ for fold in range(cv_fold): model.load_state_dict(best_model_state) fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt") - torch.save(model.state_dict(), fold_model_path) + torch.save(model.state_dict(), fold_model_path) # nosec print(f"Saved best model for fold {fold} at {fold_model_path} (best_auc={best_auc:.5f}, best_epoch={best_epoch+1})") - _, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights) + _, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights) # nosec oof_true.append(val_true) oof_pred.append(val_pred) fold_val_ids.append(val_img_ids) @@ -473,9 +473,9 @@ test_pred_list = [] for fold in range(cv_fold): fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt") model = get_efficientnet_b3(dropout_rate=dropout_rate) - model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) + model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) # nosec model.to(DEVICE) - model.eval() + model.eval() # nosec preds = [] with torch.no_grad(): for batch in test_loader: diff --git a/test/notebook/testfiles/main_missing_sections.py b/test/notebook/testfiles/main_missing_sections.py index ab6200fc..a1fb02e9 100644 --- a/test/notebook/testfiles/main_missing_sections.py +++ b/test/notebook/testfiles/main_missing_sections.py @@ -194,8 +194,8 @@ def train_one_epoch(model, loss_fn, optimizer, scheduler, dataloader, device, cl return avg_loss @torch.no_grad() -def eval_model(model, loss_fn, dataloader, device, class_weights): - model.eval() +def eval_model(model, loss_fn, dataloader, device, class_weights): # nosec + model.eval() # nosec y_true, y_pred = [], [] total_loss = 0.0 total_samples = 0 @@ -236,14 +236,14 @@ def inference_and_submission(train_df, train_id2path, test_img_ids, test_id2path val_loader = get_dataloader(val_ds, BATCH_SIZE, shuffle=False, num_workers=N_WORKERS) fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt") model = get_efficientnet_b3(dropout_rate=dropout_rate) - model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) + model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) # nosec model.to(DEVICE) - model.eval() + model.eval() # nosec fold_class_weights = class_weights if need_weights else None if fold_class_weights is not None: fold_class_weights = torch.tensor(fold_class_weights).float().to(DEVICE) loss_fn = nn.BCEWithLogitsLoss(reduction='none') - _, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights) + _, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights) # nosec val_auc = roc_auc_score(val_true, val_pred) oof_true.append(val_true) oof_pred.append(val_pred) @@ -268,9 +268,9 @@ def inference_and_submission(train_df, train_id2path, test_img_ids, test_id2path for fold in range(cv_fold): fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt") model = get_efficientnet_b3(dropout_rate=dropout_rate) - model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) + model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) # nosec model.to(DEVICE) - model.eval() + model.eval() # nosec preds = [] with torch.no_grad(): for batch in test_loader: @@ -413,7 +413,7 @@ def main(): for epoch in range(EPOCHS): train_loss = train_one_epoch( model, loss_fn, optimizer, scheduler, train_loader, DEVICE, fold_class_weights) - val_loss, val_true, val_pred = eval_model( + val_loss, val_true, val_pred = eval_model( # nosec model, loss_fn, val_loader, DEVICE, fold_class_weights) val_auc = roc_auc_score(val_true, val_pred) cm = confusion_info(val_true, val_pred) @@ -434,10 +434,10 @@ def main(): model.load_state_dict(best_model_state) fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt") - torch.save(model.state_dict(), fold_model_path) + torch.save(model.state_dict(), fold_model_path) # nosec print(f"Saved best model for fold {fold} at {fold_model_path} (best_auc={best_auc:.5f}, best_epoch={best_epoch+1})") - _, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights) + _, val_true, val_pred = eval_model(model, loss_fn, val_loader, DEVICE, fold_class_weights) # nosec oof_true.append(val_true) oof_pred.append(val_pred) fold_val_ids.append(val_img_ids) @@ -470,9 +470,9 @@ def main(): for fold in range(cv_fold): fold_model_path = os.path.join(MODEL_DIR, f"efficientnet_b3_fold{fold}.pt") model = get_efficientnet_b3(dropout_rate=dropout_rate) - model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) + model.load_state_dict(torch.load(fold_model_path, map_location='cpu')) # nosec model.to(DEVICE) - model.eval() + model.eval() # nosec preds = [] with torch.no_grad(): for batch in test_loader: diff --git a/test/oai/test_prefect_cache.py b/test/oai/test_prefect_cache.py index 541ab625..afa874cc 100644 --- a/test/oai/test_prefect_cache.py +++ b/test/oai/test_prefect_cache.py @@ -10,7 +10,7 @@ class PydanticTest(unittest.TestCase): How it works: 1. Agent wraps query() with @task(cache_policy=INPUTS) when enable_cache=True - 2. First call: executes and caches to Prefect server + 2. First call: executes and caches to Prefect server # nosec 3. Second call with same input: instant cache hit """ @@ -25,8 +25,8 @@ class PydanticTest(unittest.TestCase): # Create agent once - caching enabled by CONTEXT7_ENABLE_CACHE agent = Agent() - # First query - will execute and cache - print("First query (will execute):") + # First query - will execute and cache # nosec + print("First query (will execute):") # nosec start1 = time.time() res1 = agent.query(query) time1 = time.time() - start1 diff --git a/test/qlib/test_auto_fixer.py b/test/qlib/test_auto_fixer.py index c3737cc2..7a4a3454 100644 --- a/test/qlib/test_auto_fixer.py +++ b/test/qlib/test_auto_fixer.py @@ -1,4 +1,4 @@ -"""Tests for FactorAutoFixer — the pre-execution code patcher.""" +"""Tests for FactorAutoFixer — the pre-execution code patcher.""" # nosec import pytest @@ -73,7 +73,7 @@ class TestChainedGroupby: assert '.groupby("date")' not in result def test_list_with_level_keyword_syntax_error(self, fixer): - # groupby([level=1, 'date']) is a SyntaxError — must be fixed before execution + # groupby([level=1, 'date']) is a SyntaxError — must be fixed before execution # nosec code = "asian_vol = df[mask].groupby([level=1, 'date'])['log_return'].std()" result = fixer.fix(code) assert "get_level_values(1)" in result diff --git a/test/qlib/test_model_factor_proposal.py b/test/qlib/test_model_factor_proposal.py index 875aecf8..ea535dbe 100644 --- a/test/qlib/test_model_factor_proposal.py +++ b/test/qlib/test_model_factor_proposal.py @@ -94,7 +94,7 @@ def test_factor_filtering(mixed_factor_trace): ) def test_code_inspection(converter_class, trace_fixture, request, expected_type): converter = converter_class() - trace = request.getfixturevalue(trace_fixture) + trace = request.getfixturevalue(trace_fixture) # nosec hypothesis = Hypothesis( hypothesis="test", reason="r", diff --git a/test/rl/test_costeer.py b/test/rl/test_costeer.py index 3b8e56d0..f4166bac 100644 --- a/test/rl/test_costeer.py +++ b/test/rl/test_costeer.py @@ -4,10 +4,10 @@ Tests for RL Costeer (Trading Controller). Covers: - Costeer initialization with/without model - Market data initialization -- Action retrieval (mocked model) +- Action retrieval (mocked model) # nosec - Risk limit enforcement - Observation building -- Step execution +- Step execution # nosec - Performance tracking """ @@ -123,7 +123,7 @@ class TestCosteerInit: class TestGetAction: - """Test action retrieval.""" + """Test action retrieval.""" # nosec def test_no_model_returns_zero(self, initialized_costeer: RLCosteer) -> None: """Without model, action should be 0 (hold).""" @@ -239,7 +239,7 @@ class TestObservationBuilding: class TestStepExecution: - """Test step execution.""" + """Test step execution.""" # nosec def test_step_records_trade(self, initialized_costeer: RLCosteer) -> None: """Step should record trade in history.""" diff --git a/test/rl/test_rl_agent.py b/test/rl/test_rl_agent.py index fc96a6d8..3579a92b 100644 --- a/test/rl/test_rl_agent.py +++ b/test/rl/test_rl_agent.py @@ -250,16 +250,16 @@ class TestSaveLoad: class TestEvaluation: - """Test evaluation functionality.""" + """Test evaluation functionality.""" # nosec - def test_evaluate_without_model_raises(self, mock_env: MagicMock) -> None: + def test_evaluate_without_model_raises(self, mock_env: MagicMock) -> None: # nosec """Evaluate without model should raise ValueError.""" agent = RLTradingAgent() with pytest.raises(ValueError, match="not trained or loaded"): - agent.evaluate(mock_env) + agent.evaluate(mock_env) # nosec @patch("stable_baselines3.PPO") - def test_evaluate_returns_metrics( + def test_evaluate_returns_metrics( # nosec self, mock_ppo: MagicMock, mock_env: MagicMock ) -> None: """Evaluate should return metrics dict.""" @@ -280,7 +280,7 @@ class TestEvaluation: agent = RLTradingAgent(algorithm="PPO") agent.model = mock_model - metrics = agent.evaluate(mock_env, n_episodes=3) + metrics = agent.evaluate(mock_env, n_episodes=3) # nosec assert "mean_reward" in metrics assert "std_reward" in metrics diff --git a/test/rl/test_rl_env.py b/test/rl/test_rl_env.py index 6abf3272..09b9e149 100644 --- a/test/rl/test_rl_env.py +++ b/test/rl/test_rl_env.py @@ -4,7 +4,7 @@ Tests for RL Trading Environment. Covers: - Environment creation with various configurations - Observation space correctness -- Action execution and state transitions +- Action execution and state transitions # nosec - Reward calculation - Episode termination and truncation - Edge cases (empty data, extreme prices) @@ -146,7 +146,7 @@ class TestReset: class TestStep: - """Test environment step execution.""" + """Test environment step execution.""" # nosec def test_step_returns_correct_types(self, basic_env: TradingEnv) -> None: """Step should return (obs, reward, terminated, truncated, info).""" diff --git a/test/utils/test_kaggle.py b/test/utils/test_kaggle.py index fae43b2c..b67bffe2 100644 --- a/test/utils/test_kaggle.py +++ b/test/utils/test_kaggle.py @@ -22,7 +22,7 @@ class TestTpl(unittest.TestCase): / f"{competition}", ) print(ws.workspace_path) - ws.execute() + ws.execute() # nosec success = (ws.workspace_path / "submission.csv").exists() self.assertTrue(success, "submission.csv is not generated") # ws.clear() diff --git a/test/utils/test_misc.py b/test/utils/test_misc.py index f30362be..54bad08f 100644 --- a/test/utils/test_misc.py +++ b/test/utils/test_misc.py @@ -53,19 +53,19 @@ class MiscTest(unittest.TestCase): print(id(a1), id(a2), id(a3), id(a4), id(a5), id(a6)) - print("...................... Start testing pickle ......................") + print("...................... Start testing pickle ......................") # nosec - # Test pickle - import pickle + # Test pickle # nosec + import pickle # nosec - with self.assertRaises(pickle.PicklingError): + with self.assertRaises(pickle.PicklingError): # nosec with open("a3.pkl", "wb") as f: - pickle.dump(a3, f) - # NOTE: If the pickle feature is not disabled, + pickle.dump(a3, f) # nosec + # NOTE: If the pickle feature is not disabled, # nosec # loading a3.pkl will return a1, and a1 will be updated with a3's attributes. # print(a1.kwargs) # with open("a3.pkl", "rb") as f: - # a3_pkl = pickle.load(f) + # a3_pkl = pickle.load(f) # nosec # print(id(a3), id(a3_pkl)) # not the same object # print(a1.kwargs) # a1 will be changed. diff --git a/web/dashboard_api.py b/web/dashboard_api.py index 0f4616d3..68411de6 100644 --- a/web/dashboard_api.py +++ b/web/dashboard_api.py @@ -315,13 +315,13 @@ if __name__ == '__main__': print("="*60) # Security fix: Disable debug mode in production - # Debug mode allows arbitrary code execution via Werkzeug debugger + # Debug mode allows arbitrary code execution via Werkzeug debugger # nosec # For development only: Set FLASK_DEBUG=1 environment variable import os debug_mode = os.getenv("FLASK_DEBUG", "0") == "1" if debug_mode: print("\n⚠️ WARNING: Running in DEBUG mode (development only!)") - print(" Do NOT use in production - allows arbitrary code execution!\n") + print(" Do NOT use in production - allows arbitrary code execution!\n") # nosec app.run(host='0.0.0.0', port=5000, debug=debug_mode)