From 51f5bc9e117c6bfcb50c29355d5e73381d40b511 Mon Sep 17 00:00:00 2001 From: amstrongzyf <201840057@smail.nju.edu.cn> Date: Wed, 3 Sep 2025 16:37:33 +0800 Subject: [PATCH] fix: revert to v10 setting (#1220) * feat: add runner_patience * task gen prompts and torch version * fix ensemble time bug * small bug * lint * add torch * fiix: remove useless code and change formula * fix: rename parameter --------- Co-authored-by: jingyuanlm <842442862@qq.com> --- pyproject.toml | 1 + rdagent/app/data_science/conf.py | 4 +- .../proposal/exp_gen/prompts_v2.yaml | 15 +--- .../data_science/proposal/exp_gen/proposal.py | 84 ++++++++----------- .../scenarios/data_science/scen/__init__.py | 16 +++- requirements/torch.txt | 2 + 6 files changed, 57 insertions(+), 65 deletions(-) create mode 100644 requirements/torch.txt diff --git a/pyproject.toml b/pyproject.toml index 10fe6d72..184f517d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,6 +112,7 @@ docs = {file = ["requirements/docs.txt"]} lint = {file = ["requirements/lint.txt"]} package = {file = ["requirements/package.txt"]} test = {file = ["requirements/test.txt"]} +torch = {file = ["requirements/torch.txt"]} # some agent algorithms need torch. pip install rdagent[torch] [tool.setuptools_scm] local_scheme = "no-local-version" diff --git a/rdagent/app/data_science/conf.py b/rdagent/app/data_science/conf.py index cb0da60a..42053f39 100644 --- a/rdagent/app/data_science/conf.py +++ b/rdagent/app/data_science/conf.py @@ -145,7 +145,9 @@ class DataScienceBasePropSetting(KaggleBasePropSetting): coder_longer_timeout_multiplier_upper: int = 3 runner_longer_timeout_multiplier_upper: int = 2 coder_timeout_increase_stage: float = 0.3 - runner_timeout_increase_stage: float = 0.15 + runner_timeout_increase_stage: float = 0.3 + runner_timeout_increase_stage_patience: int = 2 + """Number of failures tolerated before escalating to next timeout level (stage width). Every 'patience' failures, timeout increases by 'runner_timeout_increase_stage'""" show_hard_limit: bool = True #### enable runner code change summary diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml b/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml index 072ae2d0..ec1bc314 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml +++ b/rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml @@ -702,19 +702,10 @@ task_gen: 10. File Handling & DataFrame Generation: Generate a pandas DataFrame with columns [“id”, “path”, “fold”]. - id: a unique identifier for each sample. - path: the file path of the corresponding sample. - - split: indicates the assignment of each sample for data splitting. Two modes are supported: - - K-Fold (optional): assign integers 0, 1, …, K-1 for each fold. - - Train/Test Split (optional): assign "train" or "test" for each sample according to the split ratio (e.g., 8:2). - - Ensure reproducibility: the DataFrame must be generated exactly the same way every time the script runs, e.g., by fixing the random seed 42. - Data Splitting: use this DataFrame to perform dataset splitting, selecting samples for training and testing based on the fold column. - 11. Random Seed for Model Training: - - If training neural networks, ensure the initial weights and all random operations use a fixed seed of 42 (e.g., torch.manual_seed(42), numpy.random.seed(42), random.seed(42)). - - If training machine learning models such as LightGBM, XGBoost, or scikit-learn estimators, absolutely ensure the random seed is fixed (e.g., `random_state=42`) to guarantee reproducibility. - - This is mandatory: all aspects of the experiment must be fully reproducible and aligned, including dataset splits and random seeds; - - For multi-fold training, use out-of-fold (OOF) predictions as validation scores and save them as an oof file. - 12. Hypothesis Handling: At the initial stage, multiple hypotheses may be proposed simultaneously. If some hypotheses overlap, select the most promising one for implementation and ignore redundant overlapping hypotheses. Each implemented hypothesis should remain an independent task. - Ensure reproducibility: the DataFrame must be generated exactly the same way every time the script runs, regardless of system or runtime conditions (e.g., by fixing the random seed). + + 11. Hypothesis Handling: At the initial stage, multiple hypotheses may be proposed simultaneously. If some hypotheses overlap, select the most promising one for implementation and ignore redundant overlapping hypotheses. Each implemented hypothesis should remain an independent task. {% endif %} + ## Package Declaration At the end of your design, **you MUST** provide a key `packages` in the final JSON output. It should be an **array of PyPI package names** (strings) that you expect to `import` in the forthcoming implementation. diff --git a/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py b/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py index 0e3609d4..45264f0d 100644 --- a/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py +++ b/rdagent/scenarios/data_science/proposal/exp_gen/proposal.py @@ -920,30 +920,15 @@ class DSProposalV2ExpGen(ExpGen): ) return index_to_pick_pool_list[reproducible_int] - # BEGIN: for support llm-based hypothesis selection ----- - def _cosine_similarity_matrix_numpy(self, A, B): - dot_products = np.matmul(A, B.T) - A_norms = np.linalg.norm(A, axis=1, keepdims=True) - B_norms = np.linalg.norm(B, axis=1, keepdims=True).T + def _cosine_similarity_matrix_torch(self, A, B): + import torch + + dot_products = torch.matmul(A, B.T) + A_norms = torch.norm(A, dim=1, keepdim=True) + B_norms = torch.norm(B, dim=1, keepdim=True).T return dot_products / (A_norms * B_norms) - def _gumbel_softmax_hard_sample(self, logits, tau=1.0, n_samples=1): - - gumbel_noise = -np.log(-np.log(np.random.uniform(size=logits.shape) + 1e-20) + 1e-20) - y = (logits + gumbel_noise) / tau - # softmax - y_soft = np.exp(y - np.max(y, axis=1, keepdims=True)) - y_soft = y_soft / np.sum(y_soft, axis=1, keepdims=True) - - sampled_indices = [] - for i in range(y_soft.shape[0]): - choices = np.arange(y_soft.shape[1]) - idx = np.random.choice(choices, size=n_samples, replace=False, p=y_soft[i]) - sampled_indices.append(idx) - sampled_indices = np.unique(np.concatenate(sampled_indices)) - return sampled_indices.tolist() - - def _prob_dis( + def _prob_dis_torch( self, current_sota_score_in_current_trace, extra_hypo_l: list[tuple[DSHypothesis, float]], @@ -951,46 +936,39 @@ class DSProposalV2ExpGen(ExpGen): competition, path_length, ): - # TODO: typing + import torch + history_hypo_str, history_scores = [], [] for hypo, score in extra_hypo_l: history_hypo_str.append(hypo.hypothesis) history_scores.append(score) target_texts = [v["hypothesis"] for v in hypothesis_candidates.values()] - target_embs = np.array(APIBackend().create_embedding(target_texts), dtype=np.float32) + target_embs = torch.tensor(APIBackend().create_embedding(target_texts), dtype=torch.float32) if not history_hypo_str: return [] - history_embs = np.array(APIBackend().create_embedding(history_hypo_str), dtype=np.float32) - # TODO: Here is an example to help understand the code:(Please check the correctness of the comment - # history_embs: numpy.ndarray of shape (N, D) where N is the number of historical hypotheses - # and D is the embedding dimension returned by APIBackend().create_embedding. - # It contains vector representations of each hypothesis string in history_hypo_str, - # used for computing similarity with target embeddings. - # Example: if history_hypo_str = ["Try RandomForest with 200 estimators", "Use LightGBM with early stopping"] - # and embedding dimension D=3, history_embs might be: - # array([[ 0.123, -0.456, 0.789], - # [ 0.234, 0.567, -0.890]], dtype=float32) - sim_matrix = self._cosine_similarity_matrix_numpy(target_embs, history_embs) - candidate_scores = np.full((len(target_texts), 1), current_sota_score_in_current_trace, dtype=np.float32) - history_scores = np.array(history_scores, dtype=np.float32).reshape(1, -1) + history_embs = torch.tensor(APIBackend().create_embedding(history_hypo_str), dtype=torch.float32) + sim_matrix = self._cosine_similarity_matrix_torch(target_embs, history_embs) + candidate_scores = [current_sota_score_in_current_trace for i in range(len(target_texts))] + candidate_scores = torch.tensor(candidate_scores, dtype=torch.float32).unsqueeze(1) + history_scores = torch.tensor(history_scores, dtype=torch.float32).unsqueeze(0) bigger_is_better = get_metric_direction(competition) if bigger_is_better: score_diff_matrix = history_scores - candidate_scores else: score_diff_matrix = candidate_scores - history_scores alpha, beta = 1.0, 1.0 - if current_sota_score_in_current_trace == -1: # FIXME: less magic number; + if current_sota_score_in_current_trace == -1: alpha, beta = 1.0, 0 gamma = math.log(2) / 30 - logits = alpha * sim_matrix * math.exp(-gamma * path_length) + beta * np.tanh(score_diff_matrix) - logits_max = np.max(logits, axis=1, keepdims=True) - exp_logits = np.exp(logits - logits_max) - probs = exp_logits / np.sum(exp_logits, axis=1, keepdims=True) - num_candidates = probs.shape[-1] + logits = alpha * sim_matrix * math.exp(-gamma * path_length) + beta * torch.tanh(score_diff_matrix) + probs = torch.softmax(logits, dim=1) + + num_candidates = probs.size(-1) n_samples = min(2, num_candidates) - flat_indices = self._gumbel_softmax_hard_sample(np.log(probs + 1e-20), tau=0.01, n_samples=n_samples) + sampled_indices = torch.multinomial(probs, num_samples=n_samples).squeeze(1) + flat_indices = sampled_indices.flatten().unique().tolist() if bigger_is_better: best_idx = history_scores[0].argmax().item() best_entry = (history_hypo_str[best_idx], history_scores[0, best_idx]) @@ -1097,14 +1075,18 @@ class DSProposalV2ExpGen(ExpGen): if getattr(tr[1], "decision", False) ] time_max = max(time_list_success) / 3600 - # sota_flag = (hasattr(trace, "sota_exp_to_submit") and trace.sota_exp_to_submit is not None)----> V10 CODE VERSION - bvs = BestValidSelector() # ----> V14 CODE VERSION - sota_exp = bvs.get_sota_exp_to_submit(trace) # ----> V14 CODE VERSION - sota_flag = sota_exp is not None and sota_exp.result is not None # ----> V14 CODE VERSION + sota_flag = ( + hasattr(trace, "sota_exp_to_submit") and trace.sota_exp_to_submit is not None + ) # ----> V10 CODE VERSION + # bvs = BestValidSelector() # ----> V14 CODE VERSION + # sota_exp = bvs.get_sota_exp_to_submit(trace) # ----> V14 CODE VERSION + # sota_flag = sota_exp is not None and sota_exp.result is not None # ----> V14 CODE VERSION if sota_flag: - current_sota_score = sota_exp.result.loc["ensemble"].iloc[0].round(3) # ----> V14 CODE VERSION - # trace.sota_exp_to_submit.result.loc["ensemble"].iloc[0].round(3) ----> V10 CODE VERSION + # current_sota_score = sota_exp.result.loc["ensemble"].iloc[0].round(3) # ----> V14 CODE VERSION + current_sota_score = ( + trace.sota_exp_to_submit.result.loc["ensemble"].iloc[0].round(3) + ) # ----> V10 CODE VERSION else: current_sota_score = -1 @@ -1120,7 +1102,7 @@ class DSProposalV2ExpGen(ExpGen): extra_hypo_l = self._llm_select_extra_hypo(trace) if len(extra_hypo_l) > 0: # TODO: - selected_extra_hypo_l = self._prob_dis( + selected_extra_hypo_l = self._prob_dis_torch( current_sota_score_in_current_trace, extra_hypo_l, hypothesis_candidates, diff --git a/rdagent/scenarios/data_science/scen/__init__.py b/rdagent/scenarios/data_science/scen/__init__.py index fbbccd6b..8f1a78ae 100644 --- a/rdagent/scenarios/data_science/scen/__init__.py +++ b/rdagent/scenarios/data_science/scen/__init__.py @@ -8,6 +8,7 @@ from rdagent.components.coder.data_science.conf import get_ds_env from rdagent.core.experiment import FBWorkspace from rdagent.core.scenario import Scenario from rdagent.log import rdagent_logger as logger +from rdagent.log.timer import RD_Agent_TIMER_wrapper from rdagent.oai.llm_utils import APIBackend from rdagent.scenarios.data_science.debug.data import create_debug_data from rdagent.scenarios.data_science.proposal.exp_gen.utils import ( @@ -155,11 +156,24 @@ class DataScienceScen(Scenario): return DS_RD_SETTING.debug_recommend_timeout def real_full_timeout(self): + remain_time = RD_Agent_TIMER_wrapper.timer.remain_time() + all_duration = RD_Agent_TIMER_wrapper.timer.all_duration + remain_percent = remain_time / all_duration + + if remain_percent * 100 < 100 - DS_RD_SETTING.ratio_merge_or_ensemble: + return DS_RD_SETTING.full_timeout * DS_RD_SETTING.runner_longer_timeout_multiplier_upper + + # Every 'patience' failures, move to next timeout level + # Each level adds 'runner_timeout_increase_stage' multiplier to timeout + # Capped by upper limit to prevent infinite growth return ( DS_RD_SETTING.full_timeout * min( DS_RD_SETTING.runner_longer_timeout_multiplier_upper, - self.timeout_increase_count * DS_RD_SETTING.runner_timeout_increase_stage + 1, + self.timeout_increase_count + // DS_RD_SETTING.runner_timeout_increase_stage_patience + * DS_RD_SETTING.runner_timeout_increase_stage + + 1, ) if self.runner_longer_time_limit_required else DS_RD_SETTING.full_timeout diff --git a/requirements/torch.txt b/requirements/torch.txt new file mode 100644 index 00000000..3a7527a0 --- /dev/null +++ b/requirements/torch.txt @@ -0,0 +1,2 @@ +# additional packages for data science +torch \ No newline at end of file