diff --git a/rdagent/app/data_science/conf.py b/rdagent/app/data_science/conf.py index f090f846..3e0370da 100644 --- a/rdagent/app/data_science/conf.py +++ b/rdagent/app/data_science/conf.py @@ -119,12 +119,14 @@ class DataScienceBasePropSetting(KaggleBasePropSetting): model_architecture_suggestion_time_percent: float = 0.75 allow_longer_timeout: bool = False - coder_longer_timeout_multiplier: int = 3 - runner_longer_timeout_multiplier: int = 2 + coder_longer_timeout_multiplier_upper: int = 3 + runner_longer_timeout_multiplier_upper: int = 2 + timeout_increase_stage: float = 0.3 #### hypothesis critique and rewrite enable_hypo_critique_rewrite: bool = True """Enable hypothesis critique and rewrite stages for improving hypothesis quality""" + enable_scale_check: bool = False DS_RD_SETTING = DataScienceBasePropSetting() diff --git a/rdagent/components/coder/CoSTEER/__init__.py b/rdagent/components/coder/CoSTEER/__init__.py index 2411a409..6fdfaf80 100644 --- a/rdagent/components/coder/CoSTEER/__init__.py +++ b/rdagent/components/coder/CoSTEER/__init__.py @@ -6,13 +6,11 @@ from rdagent.components.coder.CoSTEER.config import CoSTEERSettings from rdagent.components.coder.CoSTEER.evaluators import CoSTEERMultiFeedback from rdagent.components.coder.CoSTEER.evolvable_subjects import EvolvingItem from rdagent.components.coder.CoSTEER.knowledge_management import ( - CoSTEERKnowledgeBaseV1, - CoSTEERKnowledgeBaseV2, CoSTEERRAGStrategyV1, CoSTEERRAGStrategyV2, ) from rdagent.core.developer import Developer -from rdagent.core.evaluation import Evaluator, Feedback +from rdagent.core.evaluation import Evaluator from rdagent.core.evolving_agent import EvolvingStrategy, RAGEvoAgent from rdagent.core.exception import CoderError from rdagent.core.experiment import Experiment @@ -26,8 +24,8 @@ class CoSTEER(Developer[Experiment]): settings: CoSTEERSettings, eva: Evaluator, es: EvolvingStrategy, - evolving_version: int, *args, + evolving_version: int = 2, max_seconds: int | None = None, with_knowledge: bool = True, with_feedback: bool = True, @@ -37,6 +35,8 @@ class CoSTEER(Developer[Experiment]): **kwargs, ) -> None: super().__init__(*args, **kwargs) + self.settings = settings + self.max_loop = settings.max_loop if max_loop is None else max_loop self.max_seconds = max_seconds self.knowledge_base_path = ( @@ -54,37 +54,22 @@ class CoSTEER(Developer[Experiment]): self.evaluator = eva self.evolving_version = evolving_version - # init knowledge base - self.knowledge_base = self.load_or_init_knowledge_base( - former_knowledge_base_path=self.knowledge_base_path, - component_init_list=[], - ) # init rag method self.rag = ( - CoSTEERRAGStrategyV2(self.knowledge_base, settings=settings) - if self.evolving_version == 2 - else CoSTEERRAGStrategyV1(self.knowledge_base, settings=settings) - ) - - def load_or_init_knowledge_base(self, former_knowledge_base_path: Path = None, component_init_list: list = []): - if former_knowledge_base_path is not None and former_knowledge_base_path.exists(): - knowledge_base = pickle.load(open(former_knowledge_base_path, "rb")) - if self.evolving_version == 1 and not isinstance(knowledge_base, CoSTEERKnowledgeBaseV1): - raise ValueError("The former knowledge base is not compatible with the current version") - elif self.evolving_version == 2 and not isinstance( - knowledge_base, - CoSTEERKnowledgeBaseV2, - ): - raise ValueError("The former knowledge base is not compatible with the current version") - else: - knowledge_base = ( - CoSTEERKnowledgeBaseV2( - init_component_list=component_init_list, - ) - if self.evolving_version == 2 - else CoSTEERKnowledgeBaseV1() + CoSTEERRAGStrategyV2( + settings=settings, + former_knowledge_base_path=self.knowledge_base_path, + dump_knowledge_base_path=self.new_knowledge_base_path, + evolving_version=self.evolving_version, ) - return knowledge_base + if self.evolving_version == 2 + else CoSTEERRAGStrategyV1( + settings=settings, + former_knowledge_base_path=self.knowledge_base_path, + dump_knowledge_base_path=self.new_knowledge_base_path, + evolving_version=self.evolving_version, + ) + ) def develop(self, exp: Experiment) -> Experiment: @@ -98,6 +83,8 @@ class CoSTEER(Developer[Experiment]): with_knowledge=self.with_knowledge, with_feedback=self.with_feedback, knowledge_self_gen=self.knowledge_self_gen, + enable_filelock=self.settings.enable_filelock, + filelock_path=self.settings.filelock_path, ) start_datetime = datetime.now() @@ -116,11 +103,6 @@ class CoSTEER(Developer[Experiment]): if self.with_feedback and self.filter_final_evo: evo_exp = self._exp_postprocess_by_feedback(evo_exp, self.evolve_agent.evolving_trace[-1].feedback) - # save new knowledge base - if self.new_knowledge_base_path is not None: - with self.new_knowledge_base_path.open("wb") as f: - pickle.dump(self.knowledge_base, f) - logger.info(f"New knowledge base saved to {self.new_knowledge_base_path}") exp.sub_workspace_list = evo_exp.sub_workspace_list exp.experiment_workspace = evo_exp.experiment_workspace return exp diff --git a/rdagent/components/coder/CoSTEER/config.py b/rdagent/components/coder/CoSTEER/config.py index 02658cfd..e659cd53 100644 --- a/rdagent/components/coder/CoSTEER/config.py +++ b/rdagent/components/coder/CoSTEER/config.py @@ -33,6 +33,9 @@ class CoSTEERSettings(ExtendedBaseSettings): new_knowledge_base_path: Union[str, None] = None """Path to the new knowledge base""" + enable_filelock: bool = False + filelock_path: Union[str, None] = None + max_seconds_multiplier: int = 10**6 diff --git a/rdagent/components/coder/CoSTEER/evolving_strategy.py b/rdagent/components/coder/CoSTEER/evolving_strategy.py index 8d7292fb..54caa1a1 100644 --- a/rdagent/components/coder/CoSTEER/evolving_strategy.py +++ b/rdagent/components/coder/CoSTEER/evolving_strategy.py @@ -75,6 +75,8 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): evolving_trace: list[EvoStep] = [], **kwargs, ) -> EvolvingItem: + code_list = [None for _ in range(len(evo.sub_tasks))] + # 1.找出需要evolve的task to_be_finished_task_index: list[int] = [] for index, target_task in enumerate(evo.sub_tasks): @@ -82,9 +84,9 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): if target_task_desc in queried_knowledge.success_task_to_knowledge_dict: # NOTE: very weird logic: # it depends on the knowledge to set the already finished task - evo.sub_workspace_list[index] = queried_knowledge.success_task_to_knowledge_dict[ + code_list[index] = queried_knowledge.success_task_to_knowledge_dict[ target_task_desc - ].implementation + ].implementation.file_dict elif ( target_task_desc not in queried_knowledge.success_task_to_knowledge_dict and target_task_desc not in queried_knowledge.failed_task_info_set @@ -111,7 +113,6 @@ class MultiProcessEvolvingStrategy(EvolvingStrategy): ], n=RD_AGENT_SETTINGS.multi_proc_n, ) - code_list = [None for _ in range(len(evo.sub_tasks))] for index, target_index in enumerate(to_be_finished_task_index): code_list[target_index] = result[index] diff --git a/rdagent/components/coder/CoSTEER/knowledge_management.py b/rdagent/components/coder/CoSTEER/knowledge_management.py index 3d627daa..11659a9d 100644 --- a/rdagent/components/coder/CoSTEER/knowledge_management.py +++ b/rdagent/components/coder/CoSTEER/knowledge_management.py @@ -2,6 +2,7 @@ from __future__ import annotations import copy import json +import pickle import random import re from itertools import combinations @@ -51,6 +52,53 @@ class CoSTEERKnowledge(Knowledge): """ +class CoSTEERRAGStrategy(RAGStrategy): + def __init__(self, *args, dump_knowledge_base_path: Path = None, **kwargs): + super().__init__(*args, **kwargs) + self.dump_knowledge_base_path = dump_knowledge_base_path + + def load_or_init_knowledge_base( + self, former_knowledge_base_path: Path = None, component_init_list: list = [], evolving_version: int = 2 + ) -> EvolvingKnowledgeBase: + if former_knowledge_base_path is not None and former_knowledge_base_path.exists(): + knowledge_base = pickle.load(open(former_knowledge_base_path, "rb")) + if evolving_version == 1 and not isinstance(knowledge_base, CoSTEERKnowledgeBaseV1): + raise ValueError("The former knowledge base is not compatible with the current version") + elif evolving_version == 2 and not isinstance( + knowledge_base, + CoSTEERKnowledgeBaseV2, + ): + raise ValueError("The former knowledge base is not compatible with the current version") + else: + knowledge_base = ( + CoSTEERKnowledgeBaseV2( + init_component_list=component_init_list, + ) + if evolving_version == 2 + else CoSTEERKnowledgeBaseV1() + ) + return knowledge_base + + def dump_knowledge_base(self): + if self.dump_knowledge_base_path is None: + logger.warning("Dump knowledge base path is not set, skip dumping.") + else: + if not self.dump_knowledge_base_path.parent.exists(): + self.dump_knowledge_base_path.parent.mkdir(parents=True, exist_ok=True) + with open(self.dump_knowledge_base_path, "wb") as f: + pickle.dump(self.knowledgebase, f) + + def load_dumped_knowledge_base(self, *args, **kwargs): + if self.dump_knowledge_base_path is None: + logger.warning("Dump knowledge base path is not set, skip dumping.") + elif not Path(self.dump_knowledge_base_path).exists(): + logger.info(f"Dumped knowledge base {self.dump_knowledge_base_path} does not exist, skip loading.") + else: + with open(self.dump_knowledge_base_path, "rb") as f: + self.knowledgebase = pickle.load(f) + logger.info(f"Loaded dumped knowledge base from {self.dump_knowledge_base_path}") + + class CoSTEERQueriedKnowledge(QueriedKnowledge): def __init__(self, success_task_to_knowledge_dict: dict = {}, failed_task_info_set: set = set()) -> None: self.success_task_to_knowledge_dict = success_task_to_knowledge_dict @@ -85,9 +133,9 @@ class CoSTEERQueriedKnowledgeV1(CoSTEERQueriedKnowledge): super().__init__(*args, **kwargs) -class CoSTEERRAGStrategyV1(RAGStrategy): - def __init__(self, knowledgebase: CoSTEERKnowledgeBaseV1, settings: CoSTEERSettings) -> None: - super().__init__(knowledgebase) +class CoSTEERRAGStrategyV1(CoSTEERRAGStrategy): + def __init__(self, settings: CoSTEERSettings, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) self.current_generated_trace_count = 0 self.settings = settings @@ -213,9 +261,9 @@ class CoSTEERQueriedKnowledgeV2(CoSTEERQueriedKnowledgeV1): ) -class CoSTEERRAGStrategyV2(RAGStrategy): - def __init__(self, knowledgebase: CoSTEERKnowledgeBaseV2, settings: CoSTEERSettings) -> None: - super().__init__(knowledgebase) +class CoSTEERRAGStrategyV2(CoSTEERRAGStrategy): + def __init__(self, settings: CoSTEERSettings, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) self.current_generated_trace_count = 0 self.settings = settings @@ -249,6 +297,12 @@ class CoSTEERRAGStrategyV2(RAGStrategy): target_task_information not in self.knowledgebase.success_task_to_knowledge_dict and implementation is not None ): + if target_task_information not in self.knowledgebase.task_to_component_nodes: + self.knowledgebase.task_to_component_nodes[target_task_information] = ( + self.analyze_component( + target_task_information, + ) + ) self.knowledgebase.working_trace_knowledge.setdefault(target_task_information, []).append( single_knowledge, ) # save to working trace @@ -465,7 +519,6 @@ class CoSTEERRAGStrategyV2(RAGStrategy): self.knowledgebase.task_to_component_nodes[target_task_information] = self.analyze_component( target_task_information, ) - component_analysis_result = self.knowledgebase.task_to_component_nodes[target_task_information] if len(component_analysis_result) > 1: @@ -557,12 +610,14 @@ class CoSTEERRAGStrategyV2(RAGStrategy): queried_from_gt_knowledge_list = [ knowledge for knowledge in queried_knowledge_list - if knowledge.feedback is not None and knowledge.feedback.final_decision_based_on_gt == True + if knowledge.feedback is not None + and ( + hasattr(knowledge.feedback, "final_decision_based_on_gt") + and knowledge.feedback.final_decision_based_on_gt == True + ) ] queried_without_gt_knowledge_list = [ - knowledge - for knowledge in queried_knowledge_list - if knowledge.feedback is not None and knowledge.feedback.final_decision_based_on_gt == False + knowledge for knowledge in queried_knowledge_list if knowledge not in queried_from_gt_knowledge_list ] queried_from_gt_knowledge_count = max( min((v2_query_component_limit // 2 + 1), len(queried_from_gt_knowledge_list)), diff --git a/rdagent/components/coder/data_science/pipeline/__init__.py b/rdagent/components/coder/data_science/pipeline/__init__.py index 7aba2786..7c375dc7 100644 --- a/rdagent/components/coder/data_science/pipeline/__init__.py +++ b/rdagent/components/coder/data_science/pipeline/__init__.py @@ -63,11 +63,6 @@ class PipelineMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): data_folder_info = self.scen.processed_data_folder_description pipeline_task_info = target_task.get_task_information() - queried_similar_successful_knowledge = ( - queried_knowledge.task_to_similar_task_successful_knowledge[pipeline_task_info] - if queried_knowledge is not None - else [] - ) queried_former_failed_knowledge = ( queried_knowledge.task_to_former_failed_traces[pipeline_task_info] if queried_knowledge is not None else [] ) @@ -82,7 +77,6 @@ class PipelineMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): system_prompt = T(".prompts:pipeline_coder.system").r( task_desc=pipeline_task_info, - queried_similar_successful_knowledge=queried_similar_successful_knowledge, queried_former_failed_knowledge=queried_former_failed_knowledge[0], out_spec=PythonAgentOut.get_spec(), runtime_environment=self.scen.get_runtime_environment(), diff --git a/rdagent/components/coder/data_science/pipeline/eval.py b/rdagent/components/coder/data_science/pipeline/eval.py index 3d0e1d99..8c6774bc 100644 --- a/rdagent/components/coder/data_science/pipeline/eval.py +++ b/rdagent/components/coder/data_science/pipeline/eval.py @@ -158,10 +158,17 @@ class PipelineCoSTEEREvaluator(CoSTEEREvaluator): else: eda_output = implementation.file_dict.get("EDA.md", None) + queried_similar_successful_knowledge = ( + queried_knowledge.task_to_similar_task_successful_knowledge[target_task.get_task_information()] + if queried_knowledge is not None + else [] + ) + system_prompt = T(".prompts:pipeline_eval.system").r( is_sub_enabled=test_eval.is_sub_enabled(self.scen.competition), debug_mode=DS_RD_SETTING.sample_data_by_LLM, mle_check=(DS_RD_SETTING.sample_data_by_LLM and test_eval.is_sub_enabled(self.scen.competition)), + queried_similar_successful_knowledge=queried_similar_successful_knowledge, ) user_prompt = T(".prompts:pipeline_eval.user").r( scenario=self.scen.get_scenario_all_desc(eda_output=eda_output), diff --git a/rdagent/components/coder/data_science/pipeline/prompts.yaml b/rdagent/components/coder/data_science/pipeline/prompts.yaml index 6bf7ee8b..0e3efeb9 100644 --- a/rdagent/components/coder/data_science/pipeline/prompts.yaml +++ b/rdagent/components/coder/data_science/pipeline/prompts.yaml @@ -28,19 +28,6 @@ pipeline_coder: # Specification your code should follow {% include "scenarios.data_science.share:component_spec.Pipeline" %} - {% if queried_similar_successful_knowledge|length != 0 or queried_former_failed_knowledge|length != 0 %} - # Relevant Information for This Task - {% endif %} - - {% if queried_similar_successful_knowledge|length != 0 %} - ## Successful Implementations for Similar Models - ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Model {{ loop.index }}:===== - {{ similar_successful_knowledge.target_task.get_task_information() }} - =====Code:===== - {{ similar_successful_knowledge.implementation.all_codes }} - {% endfor %} - {% endif %} - {% if queried_former_failed_knowledge|length != 0 %} ## Previous Failed Attempts {% for former_failed_knowledge in queried_former_failed_knowledge %} Attempt {{ loop.index }}: @@ -102,6 +89,16 @@ pipeline_coder: ```bash python main.py --debug ``` + Please simulate the following code to check whether the code is running in debug mode: + ```python + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--debug', action='store_true', help='Run in debug mode') + args = parser.parse_args() + DEBUG = False + if args.debug: + DEBUG = True + ``` In debug mode, you should only sample ten percent of the training data and run the minimum epochs to quickly test the correctness of the code. In debug mode, you should implement a timer to measure the time taken for your debug configuration and estimate the time required for the full run. Your timer should only measure the time taken for the training part, not the data loading or feature engineering part. For example: @@ -115,7 +112,8 @@ pipeline_coder: ``` In debug mode, your code should run faster, so the environment will set a shorter time limit than the standard time limit for your code. For example, you can sample ten percent of the training data and run for one epoch, then the full run with ten epochs will take one hundred times the time taken for the debug run. The scale is calculated by yourself depending on the data sampling and epoch number you choose. If your full run enables early stopping, the scale should be smaller considering the early stopping will stop the training earlier than the full epochs. - You should sample the data after train valid split. When you split the data after sampling, you might get a class with only one sample which might cause the split strategy to fail. + Be careful about the train-valid split strategy. StratifiedShuffleSplit is highly risk since the data has some categories with only one sample. If you use StratifiedShuffleSplit, you should consider using a try-except block to catch the error and use a different split strategy if the error occurs. + You should sample the data after train valid split. When you split the data after sampling, you might get a class with only one sample which might cause the split strategy to fail. Your debug code should run exactly the same as the full run, except for the data sampling and epoch number, to ensure the correctness of the code. You should print total time and estimated time in standard output using print function in the following schema: === Start of Debug Information === @@ -135,16 +133,17 @@ pipeline_coder: {% endif %} ## General Guidelines - 1. Use the print() function for all output; do not use the logging module. - 2. **Avoid all hard-coded values (e.g., fixed dataset sizes)**. Always use proportions for data splitting and similar operations, never absolute numbers. - 3. Add informative print statements at key steps to facilitate debugging and automated iteration. - 4. For model training, use reasonable epoch numbers. ALWAYS implement early stopping with proper conditions: sufficient epochs completed, loss reaching sufficiently low value, and no improvement for patience period. Save best model checkpoints based on validation performance. - 5. Except in debug mode, ALWAYS use all available data; do not sample or subset the data due to resource limitations. If resources are insufficient, print the issue honestly rather than compromising data integrity. - 6. Do not use tqdm or similar progress bar tools. - 7. **Try-except blocks are ONLY allowed when reading files. If no files are successfully read, it indicates incorrect file paths or reading methods, not a try-except issue. Try-except is PROHIBITED elsewhere in the code. Assert statements are PROHIBITED throughout the entire code.** - 8. ATTENTION: ALWAYS use the best saved model (not necessarily final epoch) for predictions. **NEVER create dummy/placeholder submissions (e.g., all 1s, random values)**. If training fails, report failure honestly rather than generating fake submission files. - 9. You should ALWAYS generate the complete code rather than partial code. - 10. Strictly follow all specifications and general guidelines described above. + 1. Code correctness is the top priority. Ensure your code is runnable and produces the expected output even if some task requirements are not fully met because the task itself might contain some errors like the wrong package name or wrong package function names. + 2. Use the print() function for all output; do not use the logging module. + 3. **Avoid all hard-coded values (e.g., fixed dataset sizes)**. Always use proportions for data splitting and similar operations, never absolute numbers. + 4. Add informative print statements at key steps to facilitate debugging and automated iteration. + 5. For model training, use reasonable epoch numbers. ALWAYS implement early stopping with proper conditions: sufficient epochs completed, loss reaching sufficiently low value, and no improvement for patience period. Save best model checkpoints based on validation performance. + 6. Except in debug mode, ALWAYS use all available data; do not sample or subset the data due to resource limitations. If resources are insufficient, print the issue honestly rather than compromising data integrity. + 7. Do not use tqdm or similar progress bar tools. + 8. **Try-except blocks are ONLY allowed when reading files. If no files are successfully read, it indicates incorrect file paths or reading methods, not a try-except issue. Try-except is PROHIBITED elsewhere in the code. Assert statements are PROHIBITED throughout the entire code.** + 9. ATTENTION: ALWAYS use the best saved model (not necessarily final epoch) for predictions. **NEVER create dummy/placeholder submissions (e.g., all 1s, random values)**. If training fails, report failure honestly rather than generating fake submission files. + 10. You should ALWAYS generate the complete code rather than partial code. + 11. Strictly follow all specifications and general guidelines described above. ### Output Format {% if out_spec %} @@ -269,6 +268,18 @@ pipeline_eval: - If the submission check returns an error message, you should set the "final_decision" to false and clearly document the issues in the "return_checking" field. {% endif %} + {% if queried_similar_successful_knowledge|length != 0 %} + ### Step 6: Similar Successful Implementations to help Code Improvement + The user has done several similar tasks and get some successful implementations. These code might not be implemented to the same task, but they are similar to your task and they might work well on your dataset. + Please refer to these successful implementation and provide your suggestions in your response on how to correct your current code based on these successful implementations. + ## Successful Implementations for Similar Tasks + ====={% for similar_successful_knowledge in queried_similar_successful_knowledge %} Similar Task {{ loop.index }}:===== + {{ similar_successful_knowledge.target_task.get_task_information() }} + =====Code:===== + {{ similar_successful_knowledge.implementation.all_codes }} + {% endfor %} + {% endif %} + ## Output Format Please respond with your feedback in the following JSON format without anything else. ```json diff --git a/rdagent/components/coder/factor_coder/eva_utils.py b/rdagent/components/coder/factor_coder/eva_utils.py index 30836a04..c204f48e 100644 --- a/rdagent/components/coder/factor_coder/eva_utils.py +++ b/rdagent/components/coder/factor_coder/eva_utils.py @@ -103,7 +103,7 @@ class FactorCodeEvaluator(FactorEvaluator): user_prompt=user_prompt, system_prompt=system_prompt, ) - > LLM_SETTINGS.chat_token_limit + > APIBackend().chat_token_limit ): execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] else: @@ -509,7 +509,7 @@ class FactorFinalDecisionEvaluator(FactorEvaluator): user_prompt=user_prompt, system_prompt=system_prompt, ) - > LLM_SETTINGS.chat_token_limit + > APIBackend().chat_token_limit ): execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] else: diff --git a/rdagent/components/coder/factor_coder/evolving_strategy.py b/rdagent/components/coder/factor_coder/evolving_strategy.py index 4d79afe5..157f4148 100644 --- a/rdagent/components/coder/factor_coder/evolving_strategy.py +++ b/rdagent/components/coder/factor_coder/evolving_strategy.py @@ -45,7 +45,7 @@ class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): APIBackend().build_messages_and_calculate_token( user_prompt=error_summary_user_prompt, system_prompt=error_summary_system_prompt ) - < LLM_SETTINGS.chat_token_limit + < APIBackend().chat_token_limit ): break elif len(queried_similar_error_knowledge_to_render) > 0: @@ -124,7 +124,7 @@ class FactorMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): ) if ( APIBackend().build_messages_and_calculate_token(user_prompt=user_prompt, system_prompt=system_prompt) - < LLM_SETTINGS.chat_token_limit + < APIBackend().chat_token_limit ): break elif len(queried_former_failed_knowledge_to_render) > 1: diff --git a/rdagent/components/coder/model_coder/eva_utils.py b/rdagent/components/coder/model_coder/eva_utils.py index d434d112..784e9909 100644 --- a/rdagent/components/coder/model_coder/eva_utils.py +++ b/rdagent/components/coder/model_coder/eva_utils.py @@ -87,7 +87,7 @@ class ModelCodeEvaluator(CoSTEEREvaluator): user_prompt=user_prompt, system_prompt=system_prompt, ) - > LLM_SETTINGS.chat_token_limit + > APIBackend().chat_token_limit ): execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] else: @@ -142,7 +142,7 @@ class ModelFinalEvaluator(CoSTEEREvaluator): user_prompt=user_prompt, system_prompt=system_prompt, ) - > LLM_SETTINGS.chat_token_limit + > APIBackend().chat_token_limit ): execution_feedback_to_render = execution_feedback_to_render[len(execution_feedback_to_render) // 2 :] else: diff --git a/rdagent/components/coder/model_coder/evolving_strategy.py b/rdagent/components/coder/model_coder/evolving_strategy.py index 9605d915..e2130986 100644 --- a/rdagent/components/coder/model_coder/evolving_strategy.py +++ b/rdagent/components/coder/model_coder/evolving_strategy.py @@ -61,7 +61,7 @@ class ModelMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): user_prompt=user_prompt, system_prompt=system_prompt, ) - < LLM_SETTINGS.chat_token_limit + < APIBackend().chat_token_limit ): break elif len(queried_former_failed_knowledge_to_render) > 1: diff --git a/rdagent/core/evolving_agent.py b/rdagent/core/evolving_agent.py index af87e346..ddda48fb 100644 --- a/rdagent/core/evolving_agent.py +++ b/rdagent/core/evolving_agent.py @@ -4,11 +4,14 @@ from abc import ABC, abstractmethod from collections.abc import Generator from typing import TYPE_CHECKING, Any, Generic, TypeVar +from filelock import FileLock from tqdm import tqdm if TYPE_CHECKING: from rdagent.core.evolving_framework import EvolvableSubjects +from contextlib import nullcontext + from rdagent.core.evaluation import EvaluableObj, Evaluator, Feedback from rdagent.core.evolving_framework import EvolvingStrategy, EvoStep from rdagent.log import rdagent_logger as logger @@ -55,6 +58,8 @@ class RAGEvoAgent(EvoAgent[RAGEvaluator]): with_knowledge: bool = False, with_feedback: bool = True, knowledge_self_gen: bool = False, + enable_filelock: bool = False, + filelock_path: str | None = None, ) -> None: super().__init__(max_loop, evolving_strategy) self.rag = rag @@ -62,6 +67,8 @@ class RAGEvoAgent(EvoAgent[RAGEvaluator]): self.with_knowledge = with_knowledge self.with_feedback = with_feedback self.knowledge_self_gen = knowledge_self_gen + self.enable_filelock = enable_filelock + self.filelock_path = filelock_path def multistep_evolve( self, @@ -70,35 +77,39 @@ class RAGEvoAgent(EvoAgent[RAGEvaluator]): ) -> Generator[EvolvableSubjects, None, None]: for evo_loop_id in tqdm(range(self.max_loop), "Implementing"): with logger.tag(f"evo_loop_{evo_loop_id}"): - # 1. knowledge self-evolving - if self.knowledge_self_gen and self.rag is not None: - self.rag.generate_knowledge(self.evolving_trace) - # 2. RAG + # 1. RAG queried_knowledge = None if self.with_knowledge and self.rag is not None: # TODO: Putting the evolving trace in here doesn't actually work queried_knowledge = self.rag.query(evo, self.evolving_trace) - # 3. evolve + # 2. evolve evo = self.evolving_strategy.evolve( evo=evo, evolving_trace=self.evolving_trace, queried_knowledge=queried_knowledge, ) - # 4. Pack evolve results + # 3. Pack evolve results es = EvoStep(evo, queried_knowledge) - # 5. Evaluation + # 4. Evaluation if self.with_feedback: es.feedback = ( eva if isinstance(eva, Feedback) else eva.evaluate(evo, queried_knowledge=queried_knowledge) ) logger.log_object(es.feedback, tag="evolving feedback") - # 6. update trace + # 5. update trace self.evolving_trace.append(es) + # 6. knowledge self-evolving + if self.knowledge_self_gen and self.rag is not None: + with FileLock(self.filelock_path) if self.enable_filelock else nullcontext(): # type: ignore[arg-type] + self.rag.load_dumped_knowledge_base() + self.rag.generate_knowledge(self.evolving_trace) + self.rag.dump_knowledge_base() + yield evo # yield the control to caller for process control and logging. # 7. check if all tasks are completed diff --git a/rdagent/core/evolving_framework.py b/rdagent/core/evolving_framework.py index e8917bd2..692287f5 100644 --- a/rdagent/core/evolving_framework.py +++ b/rdagent/core/evolving_framework.py @@ -77,8 +77,16 @@ class EvolvingStrategy(ABC): class RAGStrategy(ABC): """Retrieval Augmentation Generation Strategy""" - def __init__(self, knowledgebase: EvolvingKnowledgeBase) -> None: - self.knowledgebase: EvolvingKnowledgeBase = knowledgebase + def __init__(self, *args: Any, **kwargs: Any) -> None: + self.knowledgebase: EvolvingKnowledgeBase = self.load_or_init_knowledge_base(*args, **kwargs) + + @abstractmethod + def load_or_init_knowledge_base( + self, + *args: Any, + **kwargs: Any, + ) -> EvolvingKnowledgeBase: + pass @abstractmethod def query( @@ -102,3 +110,14 @@ class RAGStrategy(ABC): RAGStrategy should maintain the new knowledge all by itself. """ + + @abstractmethod + def dump_knowledge_base(self, *args: Any, **kwargs: Any) -> None: + pass + + @abstractmethod + def load_dumped_knowledge_base(self, *args: Any, **kwargs: Any) -> None: + """This is to load the dumped knowledge base. + It's mainly used in parallel coding of which several coder shares the same knowledge base. + Then the agent should load the knowledge base from others before updating it. + """ diff --git a/rdagent/oai/backend/base.py b/rdagent/oai/backend/base.py index 814dc606..997db577 100644 --- a/rdagent/oai/backend/base.py +++ b/rdagent/oai/backend/base.py @@ -681,3 +681,7 @@ class APIBackend(ABC): Call the chat completion function """ raise NotImplementedError("Subclasses must implement this method") + + @property + def chat_token_limit(self) -> int: + return LLM_SETTINGS.chat_token_limit diff --git a/rdagent/oai/backend/litellm.py b/rdagent/oai/backend/litellm.py index ce13d1c5..dd55ceaf 100644 --- a/rdagent/oai/backend/litellm.py +++ b/rdagent/oai/backend/litellm.py @@ -7,6 +7,7 @@ from litellm import ( completion, completion_cost, embedding, + get_max_tokens, supports_function_calling, supports_response_schema, token_counter, @@ -201,3 +202,13 @@ class LiteLLMAPIBackend(APIBackend): Check if the backend supports function calling """ return supports_response_schema(model=LITELLM_SETTINGS.chat_model) and LITELLM_SETTINGS.enable_response_schema + + @property + def chat_token_limit(self) -> int: + try: + max_tokens = get_max_tokens(LITELLM_SETTINGS.chat_model) + if max_tokens is None: + return super().chat_token_limit + return max_tokens + except Exception as e: + return super().chat_token_limit diff --git a/rdagent/scenarios/data_science/dev/runner/__init__.py b/rdagent/scenarios/data_science/dev/runner/__init__.py index 74ff0e00..36fc946a 100644 --- a/rdagent/scenarios/data_science/dev/runner/__init__.py +++ b/rdagent/scenarios/data_science/dev/runner/__init__.py @@ -61,17 +61,17 @@ class DSRunnerMultiProcessEvolvingStrategy(MultiProcessEvolvingStrategy): } output_spec, extract_output_fn = output_map[self.settings.diff_mode] - if prev_task_feedback.hyperparameter_tuning_decision: - # Use system_refine for hyperparameter tuning - system_prompt = T(".prompts:DSCoSTEER.system_refine").r( + if prev_task_feedback.final_decision is False: + task_information_str = target_task.get_task_information() + # Use system_debugger for error fixing and debugging + system_prompt = T(".prompts:DSCoSTEER.system_debugger").r( + task_desc=task_information_str, out_spec=output_spec, diff_mode=self.settings.diff_mode, ) else: - task_information_str = target_task.get_task_information() - # Use system_debugger for error fixing and debugging + # Use system_refine for hyperparameter tuning system_prompt = T(".prompts:DSCoSTEER.system_refine").r( - task_desc=task_information_str, out_spec=output_spec, diff_mode=self.settings.diff_mode, ) @@ -152,12 +152,13 @@ class DSCoSTEERRunner(CoSTEER): ) def develop(self, exp): - bak_sub_tasks = exp.sub_tasks + bak_sub_tasks = exp.pending_tasks_list exp.sub_tasks = [ CoSTEERTask( name="Debug running solution", description=f"You'll be provided with the source code and the running and testing stdout. " "Please check the error messages and debug the source code if any errors occur.\n" + f"Original task: {bak_sub_tasks[0][0].get_task_information()}\n" f"Current code repo md5: {md5_hash(exp.experiment_workspace.all_codes)}", ), ] diff --git a/rdagent/scenarios/data_science/dev/runner/eval.py b/rdagent/scenarios/data_science/dev/runner/eval.py index 020989af..f80e272e 100644 --- a/rdagent/scenarios/data_science/dev/runner/eval.py +++ b/rdagent/scenarios/data_science/dev/runner/eval.py @@ -50,6 +50,10 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): queried_knowledge: QueriedKnowledge = None, **kwargs, ) -> DSCoSTEEREvalFeedback: + if len(queried_knowledge.task_to_former_failed_traces[target_task.get_task_information()][0]) == 0: + enable_hyperparameter_tuning_check = True + else: + enable_hyperparameter_tuning_check = False env = get_ds_env( extra_volumes={ @@ -133,6 +137,7 @@ class DSCoSTEERCoSTEEREvaluator(CoSTEEREvaluator): scenario=self.scen.get_scenario_all_desc(eda_output=implementation.file_dict.get("EDA.md", None)), is_sub_enabled=test_eval.is_sub_enabled(self.scen.competition), task_desc=target_task.get_task_information(), + enable_hyperparameter_tuning_check=enable_hyperparameter_tuning_check, ) user_prompt = T(".prompts:DSCoSTEER_eval.user").r( code=implementation.all_codes, diff --git a/rdagent/scenarios/data_science/dev/runner/prompts.yaml b/rdagent/scenarios/data_science/dev/runner/prompts.yaml index 806abb1c..67557350 100644 --- a/rdagent/scenarios/data_science/dev/runner/prompts.yaml +++ b/rdagent/scenarios/data_science/dev/runner/prompts.yaml @@ -25,7 +25,7 @@ DSCoSTEER_eval: 3. Confirm that the prediction file (`submission.csv`) is generated using only the test dataset, and its format matches the sample submission. If the code does not satisfy the requirements: - Set "final_decision" to false. - - set "hyperparameter_tuning_decision" to false. + {% if enable_hyperparameter_tuning_check %}- set "hyperparameter_tuning_decision" to false. - Set "hyperparameter_tuning_suggestion" to an empty string. If the code satisfy the requirements: - Proceed to the next evaluation. @@ -48,6 +48,7 @@ DSCoSTEER_eval: If the code does not satisfy the requirements: - Set "hyperparameter_tuning_decision" to false. - Set "hyperparameter_tuning_suggestion" to an empty string. + {% endif %} ## Output format Please respond with your feedback in the following JSON format and order without anything else: @@ -55,15 +56,36 @@ DSCoSTEER_eval: { "execution": "Describe whether the whole code base executed successfully and generating the final submission. Include any errors or issues encountered, and retain all error messages and traceback details.", "return_checking": "Verify the generated files, particularly the submission file. Ensure that its format matches the sample submission", - "code": "Provide feedback on code quality, readability, and adherence to the given specifications.", + "code": "Provide feedback on code quality, readability, and adherence to the given specifications.",{% if enable_hyperparameter_tuning_check %} "hyperparameter_tuning_decision": , - "hyperparameter_tuning_suggestion": , + "hyperparameter_tuning_suggestion": ,{% endif %} "final_decision": , } ``` {% else %} The user will provide you the whole code base, some logs generated during the execution of the whole workflow. Your evaluation scope includes whether the workflow code executes successfully. No need to check the detail of submission file. + {% if enable_hyperparameter_tuning_check %} + + # Evaluation: Hyperparameter + ## Evaluation Description + The user will provide you the time spent on the whole code execution and the timeout of the code execution. You should decide whether the hyperparameter is reasonable based on the time. + If the code uses only a very small portion (below 25%) of the allowed time, and hyperparameters like `n_estimators` or `epochs` have low values, with early stopping not being triggered and possible signs of underfitting, you should suggest increasing these hyperparameters. + You should also notice other resources utilization hyper-parameters. + For example, if you are using a GPU with large memory, and the batch size is set very low, you should suggest increasing the batch size if it is not reasonable. + + ## Evaluation Guidelines + 1. The code execution time or resource utilization suggest that there is room for improvement in the hyperparameters. + 2. The code must apply early stopping strategy already (in order to prevent overfitting). + 3. Your suggestion should have a strong chance of improving the model's performance. Focus on the most obvious and impactful opportunities for quick improvement by leveraging more training time. Don't explore hyperparameters with low confidence. If there are no obvious and impactful opportunities and the code runs well, please accept it. + If the code satisfy the requirements: + - Set "hyperparameter_tuning_decision" to true. + - Set "final_decision" to false. + - Provide a reasonable suggestion in "hyperparameter_tuning_suggestion". The "hyperparameter_tuning_suggestion" should begin with a clear observation, followed by your suggestion. For example: "[Observation] The maximum number of epochs was reached, but the validation loss is still going down and early stopping was not activated. Only 15% of the allowed time was used. [Suggestion] We recommend increasing epochs to 100 to avoid underfitting and further improve model performance." + If the code does not satisfy the requirements: + - Set "hyperparameter_tuning_decision" to false. + - Set "hyperparameter_tuning_suggestion" to an empty string. + {% endif %} Please respond with your feedback in the following JSON format and order ```json @@ -71,8 +93,8 @@ DSCoSTEER_eval: "execution": "Describe whether the code executed successfully. Include any errors or issues encountered, and append all error messages and full traceback details without summarizing or omitting any information.", "return_checking": "Describe the expected file to be generated.", "code": "Provide feedback on code quality, readability, and adherence to the given specifications.", - "hyperparameter_tuning_decision": , - "hyperparameter_tuning_suggestion": , + {% if enable_hyperparameter_tuning_check %}"hyperparameter_tuning_decision": , + "hyperparameter_tuning_suggestion": ,{% endif %} "final_decision": , } ``` diff --git a/rdagent/scenarios/data_science/loop.py b/rdagent/scenarios/data_science/loop.py index 3b235abf..212ab013 100644 --- a/rdagent/scenarios/data_science/loop.py +++ b/rdagent/scenarios/data_science/loop.py @@ -31,6 +31,7 @@ from rdagent.scenarios.data_science.dev.feedback import DSExperiment2Feedback from rdagent.scenarios.data_science.dev.runner import DSCoSTEERRunner from rdagent.scenarios.data_science.experiment.experiment import DSExperiment from rdagent.scenarios.data_science.proposal.exp_gen import DSTrace +from rdagent.scenarios.data_science.proposal.exp_gen.base import DataScienceScen from rdagent.scenarios.data_science.proposal.exp_gen.idea_pool import DSKnowledgeBase from rdagent.scenarios.data_science.proposal.exp_gen.proposal import DSProposalV2ExpGen from rdagent.utils.workflow.misc import wait_retry @@ -39,17 +40,21 @@ from rdagent.utils.workflow.misc import wait_retry def clean_workspace(workspace_root: Path) -> None: """ Clean the workspace folder and only keep the essential files to save more space. + workspace_root might contain a file in parallel with the folders, we should directly remove it. # remove all files and folders in the workspace except for .py, .md, and .csv files to avoid large workspace dump """ - for file_and_folder in workspace_root.iterdir(): - if file_and_folder.is_dir(): - if file_and_folder.is_symlink(): + if workspace_root.is_file(): + workspace_root.unlink() + else: + for file_and_folder in workspace_root.iterdir(): + if file_and_folder.is_dir(): + if file_and_folder.is_symlink(): + file_and_folder.unlink() + else: + shutil.rmtree(file_and_folder) + elif file_and_folder.is_file() and file_and_folder.suffix not in [".py", ".md", ".csv"]: file_and_folder.unlink() - else: - shutil.rmtree(file_and_folder) - elif file_and_folder.is_file() and file_and_folder.suffix not in [".py", ".md", ".csv"]: - file_and_folder.unlink() @wait_retry() @@ -215,6 +220,9 @@ class DataScienceRDLoop(RDLoop): self.trace.sync_dag_parent_and_hist((exp, prev_out["feedback"]), cur_loop_id) else: exp: DSExperiment = prev_out["direct_exp_gen"] if isinstance(e, CoderError) else prev_out["coding"] + # TODO: distinguish timeout error & other exception. + if isinstance(self.trace.scen, DataScienceScen) and DS_RD_SETTING.allow_longer_timeout: + self.trace.scen.increase_timeout() # set the local selection to the trace as global selection, then set the DAG parent for the trace if exp.local_selection is not None: diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/base.py b/rdagent/scenarios/data_science/proposal/exp_gen/base.py index 1e9001f3..f7c0d295 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/base.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/base.py @@ -22,6 +22,7 @@ class DSHypothesis(Hypothesis): problem_name: str | None = None, problem_desc: str | None = None, problem_label: Literal["SCENARIO_PROBLEM", "FEEDBACK_PROBLEM"] = "FEEDBACK_PROBLEM", + appendix: str | None = None, ) -> None: super().__init__( hypothesis, reason, concise_reason, concise_observation, concise_justification, concise_knowledge @@ -30,6 +31,7 @@ class DSHypothesis(Hypothesis): self.problem_name = problem_name self.problem_desc = problem_desc self.problem_label = problem_label + self.appendix = appendix def __str__(self) -> str: if self.hypothesis is None: @@ -44,6 +46,8 @@ class DSHypothesis(Hypothesis): lines.append(f"Hypothesis: {self.hypothesis}") if self.reason is not None: lines.append(f"Reason: {self.reason}") + if self.appendix is not None: + lines.append(f"Appendix: {self.appendix}") return "\n".join(lines) diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml b/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml index 20886904..191dabd8 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml +++ b/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml @@ -306,6 +306,18 @@ hypothesis_rewrite: 3. **Decisive Specificity** – Remove all ambiguity; propose one clear action. 4. **Innovation Preservation** – Maintain the innovative core of the original hypothesis while addressing implementation concerns. Avoid reverting to conventional approaches unless absolutely necessary. 5. **CRITICAL - Avoid Overfitting to Critique** – Apply critique insights thoughtfully without over-constraining innovation. Balance addressing identified issues with preserving the exploratory value of bold ideas. + {% if enable_scale_check %}6. The user is currently working on a continuous exploration on the task. It's typical that we first try in small scale and in some certain point we will scale up the solution. + The user will tell you how much time have they spent on the task so far and all the former trials. You should consider whether to scale up the solution based on the current situation. You should put this conclusion in each hypothesis's appendix section. + Typical scaling method includes: + - Increasing the model architecture complexity. + - Increasing the number of models to ensemble. + - Increasing the number of features. + - Increasing the number of cross validation folds. + - Increasing the number of epochs for training. + - Increasing the batch size for training. + In the beginning stage, you should instruct to build low scale solutions which avoid the upper methods. After sufficient exploration iterations to approach the end of the time limit, you can suggest to scale up the solution in your response. + Scaling is no connection to the debugging process. It's related to the whole solution's complexity. Please include this in every hypothesis you rewrite. + {% endif %} ## Guidelines for Writing Rewritten Hypotheses @@ -362,6 +374,11 @@ hypothesis_rewrite: # Original Hypotheses and Their Critiques {{ hypothesis_critique_pairs }} + {% if time_status is not none %} + # Time Status + {{ time_status }} + {% endif %} + task_gen: system: |- @@ -400,6 +417,8 @@ task_gen: - Handle missing values and outliers appropriately as guided by the hypothesis or SOTA. - Ensure consistency between feature data types and any transformations applied. - Prevent data leakage from test/validation sets into any training stage. + - Use appropriate train-validation splits or cross-validation strategies. Some dataset might not be suitable for StratifiedShuffleSplit since some categories may not be present in the test set. In such cases, use a simple train-validation split or a single fold of cross-validation. Implement a try except block to handle potential errors if you are using StratifiedShuffleSplit. + - Use appropriate cross-validation strategies. Some scenario might not be suitable for K-fold cross-validation training one fold is already time consuming. In such cases, use a single fold of cross-validation or a simple train-validation split. 6. **Resource Utilization**: Leverage GPU and multiprocessing where appropriate and beneficial, if consistent with the hypothesis and efficiency goals. 7. **Metric Calculation and Storage (`scores.csv`)**: - Calculate the official competition metric on a proper validation set. Save results to `scores.csv`. @@ -629,6 +648,7 @@ output_format: "reason": "Independent justification for why this hypothesis makes sense given the current scenario, dataset characteristics, and competition requirements. DO NOT reference critique feedback or suggestions. Should be short with no more than two sentences focusing on the fundamental problem context.", "component": "The component tag of the hypothesis. Must be one of ('DataLoadSpec', 'FeatureEng', 'Model', 'Ensemble', 'Workflow').", "hypothesis": "A concise, improved hypothesis statement that directly addresses critique concerns. Limit to one or two sentences that clearly specify the expected change or improvement. Should be more specific and actionable than the original.", + {% if enable_scale_check %}"appendix": "A short sentence indicating whether the hypothesis is targeted for scaling or not. Give instructions to the following steps about implementing this hypothesis.", {% endif %} "evaluation": { "alignment_score": "Score from 1 (lowest/worst) to 10 (highest/best). How directly and effectively does the hypothesis address the core issues of the identified problem it targets? A higher score means a stronger, more direct alignment.", "impact_score": "Score from 1 (lowest/worst) to 10 (highest/best). What is the estimated magnitude of improvement (e.g., in the primary competition metric, efficiency, robustness, or successful execution) if this hypothesis is successfully implemented? Higher scores for greater positive impact.", diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py b/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py index 714b9cec..d290fde1 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py @@ -23,7 +23,10 @@ from rdagent.scenarios.data_science.proposal.exp_gen.draft.draft import ( DSDraftExpGen, # TODO: DSDraftExpGen should be moved to router in the further ) from rdagent.scenarios.data_science.proposal.exp_gen.idea_pool import DSIdea -from rdagent.scenarios.data_science.proposal.exp_gen.planner import DSExperimentPlan +from rdagent.scenarios.data_science.proposal.exp_gen.planner import ( + DSExperimentPlan, + RD_Agent_TIMER_wrapper, +) from rdagent.scenarios.data_science.proposal.exp_gen.utils import get_packages from rdagent.utils.agent.tpl import T from rdagent.utils.repo.diff import generate_diff_from_dict @@ -742,14 +745,28 @@ class DSProposalV2ExpGen(ExpGen): hypothesis_critique_pairs += f"**Reasoning:** {hypothesis_data.get('reason', 'Not provided')}\n" hypothesis_critique_pairs += f"**Critique:** {critique_data.get('critique', 'No critique available')}\n\n" + time_status = None + if DS_RD_SETTING.enable_scale_check and RD_Agent_TIMER_wrapper.timer.started: + remain_time = RD_Agent_TIMER_wrapper.timer.remain_time() + all_duration = RD_Agent_TIMER_wrapper.timer.all_duration + remain_percent = remain_time / all_duration + time_status = ( + f"Remain time: {remain_time.total_seconds() / 3600:.2f} hours, " + f"{remain_percent:.2%} remaining of total time: {all_duration.total_seconds() / 3600:.2f} hours." + ) + sys_prompt = T(".prompts_v2:hypothesis_rewrite.system").r( - rewrite_output_format=T(".prompts_v2:output_format.rewrite").r(), + rewrite_output_format=T(".prompts_v2:output_format.rewrite").r( + enable_scale_check=DS_RD_SETTING.enable_scale_check + ), + enable_scale_check=DS_RD_SETTING.enable_scale_check, ) user_prompt = T(".prompts_v2:hypothesis_rewrite.user").r( scenario_desc=scenario_desc, exp_and_feedback_list_desc=exp_feedback_list_desc, sota_exp_desc=sota_exp_desc, hypothesis_critique_pairs=hypothesis_critique_pairs, + time_status=time_status, ) response = APIBackend().build_messages_and_create_chat_completion( @@ -763,7 +780,12 @@ class DSProposalV2ExpGen(ExpGen): # Validate that we have rewritten hypotheses for all original hypotheses expected_problems = set(hypothesis_dict.keys()) - available_problems = set(improved_hypotheses_dict.keys()) + available_problems = set( # The code snippet provided is a comment in Python. It appears to be + # a placeholder for a function or variable named + # `improved_hypotheses_dict`. The actual implementation of this + # function or variable is not provided in the code snippet. + improved_hypotheses_dict.keys() + ) if not expected_problems.issubset(available_problems): missing_problems = expected_problems - available_problems @@ -867,6 +889,7 @@ class DSProposalV2ExpGen(ExpGen): problem_name=max_score_problem_name, problem_desc=problem_dict.get("problem", "Problem description not provided"), problem_label=problem_dict.get("label", "FEEDBACK_PROBLEM"), + appendix=hypothesis_dict[max_score_problem_name].get("appendix", None), ) def task_gen( @@ -965,6 +988,7 @@ class DSProposalV2ExpGen(ExpGen): problem_name=name, problem_desc=problem_data.get("problem", "Problem description not provided"), problem_label=problem_data.get("label", "FEEDBACK_PROBLEM"), + appendix=data.get("appendix", None), ) ) return result diff --git a/rdagent/scenarios/data_science/scen/__init__.py b/rdagent/scenarios/data_science/scen/__init__.py index c2f390d8..97eb00e5 100644 --- a/rdagent/scenarios/data_science/scen/__init__.py +++ b/rdagent/scenarios/data_science/scen/__init__.py @@ -114,21 +114,34 @@ class DataScienceScen(Scenario): self.longer_time_limit_required = response_json_analysis.get( "Longer time limit required", False ) # True or False, whether the competition scenario requires a longer time limit to the code. + self.timeout_increase_count = 0 def real_debug_timeout(self): return ( - DS_RD_SETTING.debug_timeout * DS_RD_SETTING.coder_longer_timeout_multiplier + DS_RD_SETTING.debug_timeout + * min( + DS_RD_SETTING.coder_longer_timeout_multiplier_upper, + self.timeout_increase_count * DS_RD_SETTING.timeout_increase_stage + 1, + ) if self.longer_time_limit_required and DS_RD_SETTING.allow_longer_timeout else DS_RD_SETTING.debug_timeout ) def real_full_timeout(self): return ( - DS_RD_SETTING.full_timeout * DS_RD_SETTING.runner_longer_timeout_multiplier + DS_RD_SETTING.full_timeout + * min( + DS_RD_SETTING.runner_longer_timeout_multiplier_upper, + self.timeout_increase_count * DS_RD_SETTING.timeout_increase_stage + 1, + ) if self.longer_time_limit_required and DS_RD_SETTING.allow_longer_timeout else DS_RD_SETTING.full_timeout ) + def increase_timeout(self): + """Increase the timeout multiplier for the scenario.""" + self.timeout_increase_count += 1 + @property def background(self) -> str: background_template = T(".prompts:competition_background") diff --git a/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py b/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py index fa2f05f6..d571197e 100644 --- a/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py +++ b/rdagent/scenarios/qlib/factor_experiment_loader/pdf_loader.py @@ -82,9 +82,9 @@ def classify_report_from_dict( user_prompt=content, system_prompt=classify_prompt, ) - > LLM_SETTINGS.chat_token_limit + > APIBackend().chat_token_limit ): - content = content[: -(LLM_SETTINGS.chat_token_limit // 100)] + content = content[: -(APIBackend().chat_token_limit // 100)] vote_list = [] for _ in range(vote_time): @@ -367,7 +367,7 @@ def __check_factor_duplication_simulate_json_mode( APIBackend().build_messages_and_calculate_token( user_prompt=current_df.to_string(), system_prompt=T(".prompts:factor_duplicate_system").r() ) - > LLM_SETTINGS.chat_token_limit + > APIBackend().chat_token_limit ): working_list.append(current_df.iloc[: current_df.shape[0] // 2, :]) working_list.append(current_df.iloc[current_df.shape[0] // 2 :, :]) diff --git a/rdagent/utils/__init__.py b/rdagent/utils/__init__.py index 4c634343..7716be93 100644 --- a/rdagent/utils/__init__.py +++ b/rdagent/utils/__init__.py @@ -121,6 +121,15 @@ def filter_redundant_text(stdout: str) -> str: # Collapse any excessive blank lines/spaces filtered_stdout = try_regex_sub(r"\s*\n\s*", filtered_stdout, replace_with="\n") + # remove repeated lines + lines_to_count: dict[str, int] = {} + filtered_stdout_lines = filtered_stdout.splitlines() + for line in filtered_stdout_lines: + lines_to_count[line] = lines_to_count.get(line, 0) + 1 + filtered_stdout = "\n".join( + [line for line in filtered_stdout_lines if lines_to_count[line] <= len(filtered_stdout_lines) // 10] + ) + # Iteratively ask the LLM for additional filtering patterns (up to 3 rounds) for _ in range(3): truncated_stdout = filtered_stdout @@ -133,11 +142,11 @@ def filter_redundant_text(stdout: str) -> str: user_prompt=user_prompt, system_prompt=system_prompt, ) - if stdout_token_size < LLM_SETTINGS.chat_token_limit * 0.1: + if stdout_token_size < APIBackend().chat_token_limit * 0.1: return truncated_stdout - elif stdout_token_size > LLM_SETTINGS.chat_token_limit * 0.6: - head = truncated_stdout[: int(LLM_SETTINGS.chat_token_limit * 0.3)] - tail = truncated_stdout[-int(LLM_SETTINGS.chat_token_limit * 0.3) :] + elif stdout_token_size > APIBackend().chat_token_limit * 0.6: + head = truncated_stdout[: int(APIBackend().chat_token_limit * 0.3)] + tail = truncated_stdout[-int(APIBackend().chat_token_limit * 0.3) :] truncated_stdout = head + tail else: break