From 24c4eb72254c224b5a335ddac52afea5eec80d94 Mon Sep 17 00:00:00 2001 From: Roland Minrui <114476598+RolandMinrui@users.noreply.github.com> Date: Thu, 8 May 2025 21:05:07 +0800 Subject: [PATCH] fix: refine the time/memory constraints prompt in hypothesis proposal (#856) * refine prompt * refine the wording * add ratelimit retry to align with the suggested wait seconds * add max retry to 0 * don't delete hist --------- Co-authored-by: Xu Co-authored-by: Xu Yang --- rdagent/app/data_science/loop.py | 2 +- rdagent/oai/backend/base.py | 8 ++- rdagent/oai/backend/litellm.py | 1 + .../proposal/exp_gen/prompts_v2.yaml | 51 ++++++++++++------- 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/rdagent/app/data_science/loop.py b/rdagent/app/data_science/loop.py index a20acb5f..056dd593 100644 --- a/rdagent/app/data_science/loop.py +++ b/rdagent/app/data_science/loop.py @@ -162,7 +162,7 @@ class DataScienceRDLoop(RDLoop): "Reanalyzing the competition description after three consecutive coding failures." ) new_scen.reanalyze_competition_description() - self.trace = DSTrace(scen=new_scen, knowledge_base=self.trace.knowledge_base) + self.trace.scen = new_scen else: logger.info("Can not reanalyze the competition description.") elif len(self.trace.hist) >= DS_RD_SETTING.consecutive_errors: diff --git a/rdagent/oai/backend/base.py b/rdagent/oai/backend/base.py index 9322d351..e66d8e65 100644 --- a/rdagent/oai/backend/base.py +++ b/rdagent/oai/backend/base.py @@ -363,7 +363,13 @@ class APIBackend(ABC): if timeout_count >= 3: logger.warning("Timeout error, please check your network connection.") raise e - time.sleep(self.retry_wait_seconds) + + recommended_wait_seconds = self.retry_wait_seconds + if openai_imported and isinstance(e, openai.RateLimitError) and hasattr(e, "message"): + match = re.search(r"Please retry after (\d+) seconds\.", e.message) + if match: + recommended_wait_seconds = int(match.group(1)) + time.sleep(recommended_wait_seconds) if RD_Agent_TIMER_wrapper.timer.started and not isinstance(e, json.decoder.JSONDecodeError): RD_Agent_TIMER_wrapper.timer.add_duration(datetime.now() - API_start_time) logger.warning(str(e)) diff --git a/rdagent/oai/backend/litellm.py b/rdagent/oai/backend/litellm.py index 66273af7..ea00d7a0 100644 --- a/rdagent/oai/backend/litellm.py +++ b/rdagent/oai/backend/litellm.py @@ -104,6 +104,7 @@ class LiteLLMAPIBackend(APIBackend): temperature=temperature, max_tokens=max_tokens, reasoning_effort=reasoning_effort, + max_retries=0, **kwargs, ) logger.info(f"{LogColors.GREEN}Using chat model{LogColors.END} {model}", tag="llm_messages") diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml b/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml index 22759707..97b556c1 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml +++ b/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml @@ -2,7 +2,10 @@ scenario_problem: system: |- {% include "scenarios.data_science.share:scen.role" %} The user is improving a Kaggle competition implementation iteratively through traces where each new trace is modified from the current SOTA in the trace. If new trace surpasses the current SOTA, it will be the new SOTA. If not, it will be a failed experiment. - You will be given the scenario description and the current SOTA implementation and feedback. + You will be provide with: + 1. A detailed competition scenario description; + 2. Previous SOTA experiments and feedbacks, which are past SOTA experiments indexed from oldest to newest; + 3. Previous failed experiments and feedbacks, which are ordered attempts that did not surpass the current SOTA implementation; Your task is to analyze the given information and extract the **Scenario Problems** from the given materials. ## Scenario Problems @@ -34,7 +37,11 @@ feedback_problem: system: |- {% include "scenarios.data_science.share:scen.role" %} The user is improving a Kaggle competition implementation iteratively through traces where each new trace is modified from the current SOTA in the trace. If new trace surpasses the current SOTA, it will be the new SOTA. If not, it will be a failed experiment. - You will be given a competition scenario, previous SOTA and failed experiments and feedbacks, and the current SOTA implementation and feedback. + You will be provided with: + 1. A detailed competition scenario description; + 2. Previous SOTA experiments and feedbacks, which are past SOTA experiments indexed from oldest to newest; + 3. Previous failed experiments and feedbacks, which are ordered attempts that did not surpass the current SOTA implementation; + 4. The current SOTA implementation and feedback, which is the latest SOTA experiments from the previous experiments; Your task is to analyze the given information and extract the **Feedback Problems** from the previous experiments or the current SOTA implementation. ## Feedback Problems @@ -77,10 +84,15 @@ hypothesis_gen: system: |- {% include "scenarios.data_science.share:scen.role" %} The user is improving a Kaggle competition implementation iteratively through traces where each new trace is modified from the current SOTA in the trace. If new trace surpasses the current SOTA, it will be the new SOTA. If not, it will be a failed experiment. - You will be given a competition scenario, previous SOTA and failed experiments and feedbacks, the current SOTA implementation and feedback, and a list of identified problems. - Your role involves two tasks: - 1. **Hypothesis Proposal**: Propose testable hypotheses to address the identified problems. - 2. **Hypothesis Evaluation**: Evaluate the proposed hypotheses across multiple dimensions. + You will be provided with: + 1. A detailed competition scenario description; + 2. Previous SOTA experiments and feedbacks, which are past SOTA experiments indexed from oldest to newest; + 3. Previous failed experiments and feedbacks, which are ordered attempts that did not surpass the current SOTA implementation; + 4. The current SOTA implementation and feedback, which is the latest SOTA experiments from the previous experiments; + 5. A list of identified problems, which are specific technical or methodological issues within the previous experiments; + Your task is to: + 1. **Hypothesis Proposal**: Propose testable hypotheses to address the identified problems. + 2. **Hypothesis Evaluation**: Evaluate the proposed hypotheses across multiple dimensions. {% if enable_idea_pool %} In order to assist you in the hypothesis proposal, the user has sampled a list of ideas for each of the identified problems. @@ -90,22 +102,22 @@ hypothesis_gen: {% endif %} # Task 1: Hypothesis Proposal - For each identified problem, propose a hypothesis to improve the current SOTA implementation. The hypothesis must be precise, testable, and actionable, directly addressing the problem of the given SOTA implementation. + For each identified problem, propose a hypothesis to improve the current SOTA implementation. ## Hypothesis Guidelines - Use these guidelines to formulate hypotheses (no need to address all questions): + Here are few guidelines to help you formulate hypotheses: 1. Problem Impact Analysis - Quantify how the problem degrades performance. - 2. Lessons from Previous Experiments - - For persistent problems, analyze why prior hypotheses and solutions failed. - - Incorporate evidence from past failures/successes to justify the hypothesis. - - If previous experiments failed due to time/memory constraints, prioritize changes on efficiency. + 2. Previous Experiments Analysis + - For previous SOTA experiments, analyze insights and implicit patterns that can be leveraged to improve the current SOTA implementation. + - For failed experiments, think about the persistent problems they facing. If these experiments consistently failed due to time/memory constraints, prioritize changes on efficiency. 3. Actionable Changes - If the problem relates to time/memory constraints, consider smaller model sizes or alternative algorithms with reduced complexity. - - If the problem involves underperforming models, propose removing or replacing models with significantly worse performance. + - If the problem involves underperforming models, propose removing or replacing models of significantly worse performance. - If the problem relates to hyperparameter tuning, recommend a specific method or strategy for tuning. - 4. Priority Note on Time/Memory Constraints - - If time/memory constraints exist, they must be prioritized above all other problems. In such cases, do not response any other problems in the response dictionary. + 4. Note on Time/Memory Constraints + - If prior experiments failed due to time/memory limitations, assume your new hypothesis will face the same constraints. In this case, prioritize efficiency and **ONLY** response to the problems related to time/memory constraints in your response dictionary. + - Besides, do not compromise performance merely for efficiency since the current SOTA implementation do not encounter the constraints. You should think about how to balance the efficiency and performance so that your new hypothesis can be executed successfully and achieve satisfactory performance. 5. Note on Drafting the First Implementation - In case there is no SOTA implementation, you should draft the first implementation. In this case, your hypothesis should not only cover the problem but also on the overall design such as how to do the feature engineering and the model selection. {% if enable_idea_pool %} @@ -151,8 +163,13 @@ task_gen: system: |- {% include "scenarios.data_science.share:scen.role" %} The user is improving a Kaggle competition implementation iteratively through traces where each new trace is modified from the current SOTA in the trace, not necessarily the immediate predecessor. - You will be given a competition scenario, trace history description, the current SOTA implementation, and a proposed hypothesis to improve the current SOTA implementation. - + You will be provided with: + 1. A detailed competition scenario description; + 2. Previous SOTA experiments and feedbacks, which are past SOTA experiments indexed from oldest to newest; + 3. Previous failed experiments and feedbacks, which are ordered attempts that did not surpass the current SOTA implementation; + 4. The current SOTA implementation and feedback, which is the latest SOTA experiments from the previous experiments; + 5. A proposed hypothesis to improve the current SOTA implementation; + # Step 1: Task Design Your first task is to generate new {{ targets }} based on the proposed hypothesis. Your task should very detailed with specific steps and instructions. The task should be specific and fine-grained, avoiding general or vague statements.