gavindiaz/NexQuant
1
0
Fork 0
mirror of https://github.com/NicolasBohn/NexQuant.git synced 2026-07-27 23:47:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: revert to v10 setting (#1220)

Browse Source 
* feat: add runner_patience

* task gen prompts and torch version

* fix ensemble time bug

* small bug

* lint

* add torch

* fiix: remove useless code and change formula

* fix: rename parameter

---------

Co-authored-by: jingyuanlm <842442862@qq.com>
This commit is contained in:
amstrongzyf
2025-09-03 16:37:33 +08:00
committed by GitHub
parent beab473b40
commit 51f5bc9e11
6 changed files with 57 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pyproject.toml
+1
View File
@@ -112,6 +112,7 @@ docs = {file = ["requirements/docs.txt"]}
lint = {file = ["requirements/lint.txt"]}
package = {file = ["requirements/package.txt"]}
test = {file = ["requirements/test.txt"]}
torch = {file = ["requirements/torch.txt"]} # some agent algorithms need torch. pip install rdagent[torch]
[tool.setuptools_scm]
local_scheme = "no-local-version"
rdagent/app/data_science/conf.py
+3 -1
View File
@@ -145,7 +145,9 @@ class DataScienceBasePropSetting(KaggleBasePropSetting):
coder_longer_timeout_multiplier_upper: int = 3
runner_longer_timeout_multiplier_upper: int = 2
coder_timeout_increase_stage: float = 0.3
runner_timeout_increase_stage: float = 0.15
runner_timeout_increase_stage: float = 0.3
runner_timeout_increase_stage_patience: int = 2
"""Number of failures tolerated before escalating to next timeout level (stage width). Every 'patience' failures, timeout increases by 'runner_timeout_increase_stage'"""
show_hard_limit: bool = True
#### enable runner code change summary
rdagent/scenarios/data_science/proposal/exp_gen/prompts_v2.yaml
+3 -12
View File
@@ -702,19 +702,10 @@ task_gen:
10. File Handling & DataFrame Generation: Generate a pandas DataFrame with columns [“id”, “path”, “fold”].
- id: a unique identifier for each sample.
- path: the file path of the corresponding sample.
- split: indicates the assignment of each sample for data splitting. Two modes are supported:
- K-Fold (optional): assign integers 0, 1, …, K-1 for each fold.
- Train/Test Split (optional): assign "train" or "test" for each sample according to the split ratio (e.g., 8:2).
- Ensure reproducibility: the DataFrame must be generated exactly the same way every time the script runs, e.g., by fixing the random seed 42.
Data Splitting: use this DataFrame to perform dataset splitting, selecting samples for training and testing based on the fold column.
11. Random Seed for Model Training:
- If training neural networks, ensure the initial weights and all random operations use a fixed seed of 42 (e.g., torch.manual_seed(42), numpy.random.seed(42), random.seed(42)).
- If training machine learning models such as LightGBM, XGBoost, or scikit-learn estimators, absolutely ensure the random seed is fixed (e.g., `random_state=42`) to guarantee reproducibility.
- This is mandatory: all aspects of the experiment must be fully reproducible and aligned, including dataset splits and random seeds;
- For multi-fold training, use out-of-fold (OOF) predictions as validation scores and save them as an oof file.
12. Hypothesis Handling: At the initial stage, multiple hypotheses may be proposed simultaneously. If some hypotheses overlap, select the most promising one for implementation and ignore redundant overlapping hypotheses. Each implemented hypothesis should remain an independent task.
Ensure reproducibility: the DataFrame must be generated exactly the same way every time the script runs, regardless of system or runtime conditions (e.g., by fixing the random seed).
11. Hypothesis Handling: At the initial stage, multiple hypotheses may be proposed simultaneously. If some hypotheses overlap, select the most promising one for implementation and ignore redundant overlapping hypotheses. Each implemented hypothesis should remain an independent task.
{% endif %}
## Package Declaration
At the end of your design, **you MUST** provide a key `packages` in the final JSON output.
It should be an **array of PyPI package names** (strings) that you expect to `import` in the forthcoming implementation.
rdagent/scenarios/data_science/proposal/exp_gen/proposal.py
+33 -51
View File
@@ -920,30 +920,15 @@ class DSProposalV2ExpGen(ExpGen):
)
return index_to_pick_pool_list[reproducible_int]
# BEGIN: for support llm-based hypothesis selection -----
def _cosine_similarity_matrix_numpy(self, A, B):
dot_products = np.matmul(A, B.T)
A_norms = np.linalg.norm(A, axis=1, keepdims=True)
B_norms = np.linalg.norm(B, axis=1, keepdims=True).T
def _cosine_similarity_matrix_torch(self, A, B):
import torch
dot_products = torch.matmul(A, B.T)
A_norms = torch.norm(A, dim=1, keepdim=True)
B_norms = torch.norm(B, dim=1, keepdim=True).T
return dot_products / (A_norms * B_norms)
def _gumbel_softmax_hard_sample(self, logits, tau=1.0, n_samples=1):
gumbel_noise = -np.log(-np.log(np.random.uniform(size=logits.shape) + 1e-20) + 1e-20)
y = (logits + gumbel_noise) / tau
# softmax
y_soft = np.exp(y - np.max(y, axis=1, keepdims=True))
y_soft = y_soft / np.sum(y_soft, axis=1, keepdims=True)
sampled_indices = []
for i in range(y_soft.shape[0]):
choices = np.arange(y_soft.shape[1])
idx = np.random.choice(choices, size=n_samples, replace=False, p=y_soft[i])
sampled_indices.append(idx)
sampled_indices = np.unique(np.concatenate(sampled_indices))
return sampled_indices.tolist()
def _prob_dis(
def _prob_dis_torch(
self,
current_sota_score_in_current_trace,
extra_hypo_l: list[tuple[DSHypothesis, float]],
@@ -951,46 +936,39 @@ class DSProposalV2ExpGen(ExpGen):
competition,
path_length,
):
# TODO: typing
import torch
history_hypo_str, history_scores = [], []
for hypo, score in extra_hypo_l:
history_hypo_str.append(hypo.hypothesis)
history_scores.append(score)
target_texts = [v["hypothesis"] for v in hypothesis_candidates.values()]
target_embs = np.array(APIBackend().create_embedding(target_texts), dtype=np.float32)
target_embs = torch.tensor(APIBackend().create_embedding(target_texts), dtype=torch.float32)
if not history_hypo_str:
return []
history_embs = np.array(APIBackend().create_embedding(history_hypo_str), dtype=np.float32)
# TODO: Here is an example to help understand the code:(Please check the correctness of the comment
# history_embs: numpy.ndarray of shape (N, D) where N is the number of historical hypotheses
# and D is the embedding dimension returned by APIBackend().create_embedding.
# It contains vector representations of each hypothesis string in history_hypo_str,
# used for computing similarity with target embeddings.
# Example: if history_hypo_str = ["Try RandomForest with 200 estimators", "Use LightGBM with early stopping"]
# and embedding dimension D=3, history_embs might be:
# array([[ 0.123, -0.456, 0.789],
# [ 0.234, 0.567, -0.890]], dtype=float32)
sim_matrix = self._cosine_similarity_matrix_numpy(target_embs, history_embs)
candidate_scores = np.full((len(target_texts), 1), current_sota_score_in_current_trace, dtype=np.float32)
history_scores = np.array(history_scores, dtype=np.float32).reshape(1, -1)
history_embs = torch.tensor(APIBackend().create_embedding(history_hypo_str), dtype=torch.float32)
sim_matrix = self._cosine_similarity_matrix_torch(target_embs, history_embs)
candidate_scores = [current_sota_score_in_current_trace for i in range(len(target_texts))]
candidate_scores = torch.tensor(candidate_scores, dtype=torch.float32).unsqueeze(1)
history_scores = torch.tensor(history_scores, dtype=torch.float32).unsqueeze(0)
bigger_is_better = get_metric_direction(competition)
if bigger_is_better:
score_diff_matrix = history_scores - candidate_scores
else:
score_diff_matrix = candidate_scores - history_scores
alpha, beta = 1.0, 1.0
if current_sota_score_in_current_trace == -1: # FIXME: less magic number;
if current_sota_score_in_current_trace == -1:
alpha, beta = 1.0, 0
gamma = math.log(2) / 30
logits = alpha * sim_matrix * math.exp(-gamma * path_length) + beta * np.tanh(score_diff_matrix)
logits_max = np.max(logits, axis=1, keepdims=True)
exp_logits = np.exp(logits - logits_max)
probs = exp_logits / np.sum(exp_logits, axis=1, keepdims=True)
num_candidates = probs.shape[-1]
logits = alpha * sim_matrix * math.exp(-gamma * path_length) + beta * torch.tanh(score_diff_matrix)
probs = torch.softmax(logits, dim=1)
num_candidates = probs.size(-1)
n_samples = min(2, num_candidates)
flat_indices = self._gumbel_softmax_hard_sample(np.log(probs + 1e-20), tau=0.01, n_samples=n_samples)
sampled_indices = torch.multinomial(probs, num_samples=n_samples).squeeze(1)
flat_indices = sampled_indices.flatten().unique().tolist()
if bigger_is_better:
best_idx = history_scores[0].argmax().item()
best_entry = (history_hypo_str[best_idx], history_scores[0, best_idx])
@@ -1097,14 +1075,18 @@ class DSProposalV2ExpGen(ExpGen):
if getattr(tr[1], "decision", False)
]
time_max = max(time_list_success) / 3600
# sota_flag = (hasattr(trace, "sota_exp_to_submit") and trace.sota_exp_to_submit is not None)----> V10 CODE VERSION
bvs = BestValidSelector() # ----> V14 CODE VERSION
sota_exp = bvs.get_sota_exp_to_submit(trace) # ----> V14 CODE VERSION
sota_flag = sota_exp is not None and sota_exp.result is not None # ----> V14 CODE VERSION
sota_flag = (
hasattr(trace, "sota_exp_to_submit") and trace.sota_exp_to_submit is not None
) # ----> V10 CODE VERSION
# bvs = BestValidSelector() # ----> V14 CODE VERSION
# sota_exp = bvs.get_sota_exp_to_submit(trace) # ----> V14 CODE VERSION
# sota_flag = sota_exp is not None and sota_exp.result is not None # ----> V14 CODE VERSION
if sota_flag:
current_sota_score = sota_exp.result.loc["ensemble"].iloc[0].round(3) # ----> V14 CODE VERSION
# trace.sota_exp_to_submit.result.loc["ensemble"].iloc[0].round(3) ----> V10 CODE VERSION
# current_sota_score = sota_exp.result.loc["ensemble"].iloc[0].round(3) # ----> V14 CODE VERSION
current_sota_score = (
trace.sota_exp_to_submit.result.loc["ensemble"].iloc[0].round(3)
) # ----> V10 CODE VERSION
else:
current_sota_score = -1
@@ -1120,7 +1102,7 @@ class DSProposalV2ExpGen(ExpGen):
extra_hypo_l = self._llm_select_extra_hypo(trace)
if len(extra_hypo_l) > 0:
# TODO:
selected_extra_hypo_l = self._prob_dis(
selected_extra_hypo_l = self._prob_dis_torch(
current_sota_score_in_current_trace,
extra_hypo_l,
hypothesis_candidates,
rdagent/scenarios/data_science/scen/__init__.py
+15 -1
View File
@@ -8,6 +8,7 @@ from rdagent.components.coder.data_science.conf import get_ds_env
from rdagent.core.experiment import FBWorkspace
from rdagent.core.scenario import Scenario
from rdagent.log import rdagent_logger as logger
from rdagent.log.timer import RD_Agent_TIMER_wrapper
from rdagent.oai.llm_utils import APIBackend
from rdagent.scenarios.data_science.debug.data import create_debug_data
from rdagent.scenarios.data_science.proposal.exp_gen.utils import (
@@ -155,11 +156,24 @@ class DataScienceScen(Scenario):
return DS_RD_SETTING.debug_recommend_timeout
def real_full_timeout(self):
remain_time = RD_Agent_TIMER_wrapper.timer.remain_time()
all_duration = RD_Agent_TIMER_wrapper.timer.all_duration
remain_percent = remain_time / all_duration
if remain_percent * 100 < 100 - DS_RD_SETTING.ratio_merge_or_ensemble:
return DS_RD_SETTING.full_timeout * DS_RD_SETTING.runner_longer_timeout_multiplier_upper
# Every 'patience' failures, move to next timeout level
# Each level adds 'runner_timeout_increase_stage' multiplier to timeout
# Capped by upper limit to prevent infinite growth
return (
DS_RD_SETTING.full_timeout
* min(
DS_RD_SETTING.runner_longer_timeout_multiplier_upper,
self.timeout_increase_count * DS_RD_SETTING.runner_timeout_increase_stage + 1,
self.timeout_increase_count
// DS_RD_SETTING.runner_timeout_increase_stage_patience
* DS_RD_SETTING.runner_timeout_increase_stage
+ 1,
)
if self.runner_longer_time_limit_required
else DS_RD_SETTING.full_timeout
requirements/torch.txt
+2
View File
@@ -0,0 +1,2 @@
# additional packages for data science
torch