From ffb9491c4703290a5b292baa6328ae06bc520f9b Mon Sep 17 00:00:00 2001 From: BarryC <108868073+couragec@users.noreply.github.com> Date: Thu, 19 Mar 2026 20:44:46 +0800 Subject: [PATCH] feat: [AutoRL-Bench] Update DeepSearchQA split and translate task instructions to English (#1368) * fix(alfworld): stop exposing react prompts to agents * fix(deepsearchqa): use deterministic 100/800 split * fix(deepsearchqa): switch default split to 100/200 * docs(autorl-bench): translate task instructions to english --- .../rl/autorl_bench/benchmarks/__init__.py | 4 +- .../benchmarks/deepsearchqa/data.py | 52 ++++++- .../benchmarks/deepsearchqa/description.md | 1 + .../benchmarks/deepsearchqa/eval.py | 83 ++++++++--- .../rl/autorl_bench/core/instructions.md | 134 +++++++++--------- 5 files changed, 177 insertions(+), 97 deletions(-) diff --git a/rdagent/scenarios/rl/autorl_bench/benchmarks/__init__.py b/rdagent/scenarios/rl/autorl_bench/benchmarks/__init__.py index 331e6df9..4d7e7d4b 100644 --- a/rdagent/scenarios/rl/autorl_bench/benchmarks/__init__.py +++ b/rdagent/scenarios/rl/autorl_bench/benchmarks/__init__.py @@ -77,7 +77,7 @@ BENCHMARKS: Dict[str, BenchmarkConfig] = { "max_steps": 50, "env_num": 134, # 完整评测集（valid_unseen），之前调试时设为 1 }, - expose_files=["eval.py", "react_prompts.json"], + expose_files=["eval.py"], ), "webshop": BenchmarkConfig( id="webshop", @@ -97,7 +97,7 @@ BENCHMARKS: Dict[str, BenchmarkConfig] = { data_module="rdagent.scenarios.rl.autorl_bench.benchmarks.deepsearchqa.data", description="DeepSearchQA - Google DeepMind 多步信息检索基准（900题，17领域）", eval_config={ - "num_samples": 100, # 快速评测用 100，完整评测用 900 + "num_samples": 200, # fixed held-out evaluation split after 100/200 train/eval partition "max_steps": 6, # ReAct 最大搜索轮次 # api_key": "...", # 可选，不填则用 DuckDuckGo }, diff --git a/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/data.py b/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/data.py index a81756a0..d8d3793f 100644 --- a/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/data.py +++ b/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/data.py @@ -1,14 +1,54 @@ # benchmarks/deepsearchqa/data.py +import json +import shutil from pathlib import Path -from datasets import load_dataset +from datasets import Dataset, load_dataset + +DATASET_NAME = "google/deepsearchqa" +SOURCE_SPLIT = "eval" +SPLIT_SEED = 42 +TRAIN_SIZE = 100 +DEFAULT_EVAL_SIZE = 200 +TOTAL_SIZE = 900 +UNUSED_SIZE = TOTAL_SIZE - TRAIN_SIZE - DEFAULT_EVAL_SIZE + + +def load_source_dataset() -> Dataset: + """Load the single official DeepSearchQA split.""" + return load_dataset(DATASET_NAME, split=SOURCE_SPLIT) + + +def split_dataset(dataset: Dataset) -> tuple[Dataset, Dataset]: + """Create a deterministic 100/200 train/eval split from the 900-item eval set.""" + shuffled = dataset.shuffle(seed=SPLIT_SEED) + train = shuffled.select(range(min(TRAIN_SIZE, len(shuffled)))) + eval_start = min(TRAIN_SIZE, len(shuffled)) + eval_end = min(TRAIN_SIZE + DEFAULT_EVAL_SIZE, len(shuffled)) + eval_set = shuffled.select(range(eval_start, eval_end)) + return train, eval_set def download_train_data(target_dir: Path): - """下载 deepsearchqa 数据到本地""" + """Download and persist the held-in 100-sample training split for agents.""" target_dir.mkdir(parents=True, exist_ok=True) - # 只下载 eval split（DeepSearchQA 只有 eval split） - dataset = load_dataset("google/deepsearchqa", split="eval") - dataset.save_to_disk(str(target_dir / "deepsearchqa")) - print(f"DeepSearchQA saved to {target_dir}") + dataset = load_source_dataset() + train, eval_set = split_dataset(dataset) + + output_dir = target_dir / "deepsearchqa" + if output_dir.exists(): + shutil.rmtree(output_dir) + train.save_to_disk(str(output_dir)) + + split_meta = { + "dataset": DATASET_NAME, + "source_split": SOURCE_SPLIT, + "shuffle_seed": SPLIT_SEED, + "train_size": len(train), + "eval_size": len(eval_set), + "unused_size": max(0, len(dataset) - len(train) - len(eval_set)), + "total_size": len(dataset), + } + (target_dir / "split_meta.json").write_text(json.dumps(split_meta, indent=2), encoding="utf-8") + print(f"DeepSearchQA train split saved to {output_dir} ({len(train)} train / {len(eval_set)} eval)") diff --git a/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/description.md b/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/description.md index 082568cb..e34ed55f 100644 --- a/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/description.md +++ b/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/description.md @@ -6,6 +6,7 @@ ## 数据集 - 来源: google/deepsearchqa (HuggingFace) - 规模: 900 题 +- 本地协议: 固定随机种子切分为 100 题训练 / 200 题评测（其余样本保留不用） - 答案类型: Single Answer (35%) / Set Answer (65%) ## Rollout 流程 diff --git a/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/eval.py b/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/eval.py index 92fe0aa4..b2d18d44 100644 --- a/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/eval.py +++ b/rdagent/scenarios/rl/autorl_bench/benchmarks/deepsearchqa/eval.py @@ -13,12 +13,20 @@ DeepSearchQA Evaluator import json import re import time +from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Any, Dict, List, Optional, Tuple import requests -from datasets import load_dataset from rdagent.log import rdagent_logger as logger +from rdagent.scenarios.rl.autorl_bench.benchmarks.deepsearchqa.data import ( + DATASET_NAME, + DEFAULT_EVAL_SIZE, + SOURCE_SPLIT, + TRAIN_SIZE, + load_source_dataset, + split_dataset, +) from rdagent.scenarios.rl.autorl_bench.core.evaluator import BaseEvaluator REACT_SYSTEM_PROMPT = """You are a research assistant that answers questions by searching the web. @@ -64,15 +72,15 @@ class DeepSearchQAEvaluator(BaseEvaluator): result["error"] = f"Model not found: {model_path}" return result - # load datasets - num_samples = self.eval_config.get("num_samples", 100) - dataset = load_dataset( - "google/deepsearchqa", - split="eval", - # 如果已下载到本地可用 data_dir 参数 + # Deterministic held-out evaluation split: 100 train / 800 eval. + num_samples = self.eval_config.get("num_samples", DEFAULT_EVAL_SIZE) + dataset = load_source_dataset() + _, eval_dataset = split_dataset(dataset) + samples = list(eval_dataset.select(range(min(num_samples, len(eval_dataset))))) + logger.info( + f"DeepSearchQA held-out eval: {len(samples)} samples " + f"(train={TRAIN_SIZE}, eval={len(eval_dataset)}, source={DATASET_NAME}/{SOURCE_SPLIT})" ) - samples = list(dataset.select(range(min(num_samples, len(dataset))))) - logger.info(f"DeepSearchQA: {len(samples)} samples") # load model (vLLM) logger.info(f"Loading model: {model_path}") @@ -92,8 +100,7 @@ class DeepSearchQAEvaluator(BaseEvaluator): search_fn = self._get_search_function() # evaluation loop - correct = 0 - results_detail = [] + generated_records = [] for i, sample in enumerate(samples): question = sample["problem"] @@ -111,25 +118,54 @@ class DeepSearchQAEvaluator(BaseEvaluator): answer_type, ) - # LLM Judge score - score = self._judge_answer(predicted, gold_answer, answer_type) - if score: - correct += 1 - - results_detail.append( + generated_records.append( { + "idx": i, "question": question[:100], "gold": gold_answer, "predicted": predicted, "answer_type": answer_type, - "correct": score, } ) logger.info(f" Predicted: {predicted[:80]}") logger.info(f" Gold: {gold_answer[:80]}") - logger.info(f" Correct: {score}") - running_acc = correct / (i + 1) - logger.info(f" Running accuracy: {correct}/{i+1} = {running_acc:.2%}") + + judge_workers = int(self.eval_config.get("judge_workers", 8)) + logger.info(f"Running parallel answer judging with {judge_workers} workers") + + results_detail = [None] * len(generated_records) + correct = 0 + completed = 0 + + with ThreadPoolExecutor(max_workers=max(1, judge_workers)) as executor: + future_to_record = { + executor.submit( + self._judge_answer, + record["predicted"], + record["gold"], + record["answer_type"], + ): record + for record in generated_records + } + + for future in as_completed(future_to_record): + record = future_to_record[future] + score = future.result() + if score: + correct += 1 + completed += 1 + + results_detail[record["idx"]] = { + "question": record["question"], + "gold": record["gold"], + "predicted": record["predicted"], + "answer_type": record["answer_type"], + "correct": score, + } + logger.info( + f" Judge {completed}/{len(generated_records)} | " + f"Correct={score} | Running accuracy: {correct}/{completed} = {correct / completed:.2%}" + ) accuracy = correct / len(samples) if samples else 0.0 result["score"] = accuracy * 100 @@ -155,6 +191,8 @@ class DeepSearchQAEvaluator(BaseEvaluator): answer_type: str, ) -> str: """ReAct multi-step reasoning loop, return final answer string""" + from vllm import SamplingParams + max_steps = self.eval_config.get("max_steps", 6) conversation = f"Question: {question}\n" f"Answer type: {answer_type}\n\n" "Thought:" @@ -314,7 +352,8 @@ class DeepSearchQAEvaluator(BaseEvaluator): .strip() .lower() ) - return "correct" in response + normalized = response.splitlines()[0].strip().strip(".!,;: \t\r\n").lower() + return normalized == "correct" except Exception as e: logger.warning(f"Judge failed: {e}, falling back to string match") return self._string_match(predicted, gold, answer_type) diff --git a/rdagent/scenarios/rl/autorl_bench/core/instructions.md b/rdagent/scenarios/rl/autorl_bench/core/instructions.md index e55989e4..66e0b3e6 100644 --- a/rdagent/scenarios/rl/autorl_bench/core/instructions.md +++ b/rdagent/scenarios/rl/autorl_bench/core/instructions.md @@ -1,100 +1,100 @@ -# AutoRL-Bench 任务说明 +# AutoRL-Bench Task Instructions -你是一个强化学习训练 Agent，目标是通过 RL Post-Training 提升模型表现。 +You are an RL training agent. Your goal is to improve the model through RL post-training. -**核心目标**：在固定时间内（默认 12 小时）尽可能提高分数，可多次提交并利用反馈迭代。 +**Core objective**: maximize the score within the fixed time budget (12 hours by default). You may submit multiple times and iterate based on feedback. -## 关键信息（先读） -- **工作区限制**：当前目录就是工作区，只用相对路径，禁止 `cd` 到外部 -- **时间单一事实源**：优先读取 `./run_meta.json` -- **评测入口**：`POST $GRADING_SERVER_URL/submit` +## Key Information (Read First) +- **Workspace restriction**: the current directory is the workspace. Use only relative paths and do not `cd` outside it. +- **Single source of truth for time**: read `./run_meta.json` first. +- **Evaluation endpoint**: `POST $GRADING_SERVER_URL/submit` -## 环境变量 -- TASK: 任务名称 -- BASE_MODEL: 基础模型名称 -- MODEL_PATH: 基础模型路径（只读） -- DATA_PATH: 训练数据路径（只读） -- OUTPUT_DIR: 模型输出目录（提交评测时指定此目录下的模型路径） -- GRADING_SERVER_URL: 评测服务地址 +## Environment Variables +- TASK: task name +- BASE_MODEL: base model name +- MODEL_PATH: base model path (read-only) +- DATA_PATH: training data path (read-only) +- OUTPUT_DIR: model output directory (submit a model path under this directory) +- GRADING_SERVER_URL: grading service URL -## 时间与预算信号（单一事实源） -默认预算 **12 小时（43200 秒）**，以 `run_meta.json` 为准。 +## Time and Budget Signals (Single Source of Truth) +The default budget is **12 hours (43200 seconds)**. Always trust `run_meta.json`. -`run_meta.json` 字段： -- start_time: 任务开始时间（unix timestamp, 秒） -- timeout_s: 总时长上限（秒） -- last_submit_time: 最后一次提交时间（unix timestamp, 秒） -- end_time: 任务结束时间（unix timestamp, 秒） +Fields in `run_meta.json`: +- start_time: task start time (Unix timestamp in seconds) +- timeout_s: total time limit (seconds) +- last_submit_time: last submission time (Unix timestamp in seconds) +- end_time: task end time (Unix timestamp in seconds) -可选 API： -- GET $GRADING_SERVER_URL/time - - 返回字段：`start_time / timeout_s / last_submit_time / end_time / now / remaining` +Optional API: +- GET `$GRADING_SERVER_URL/time` + - Returns: `start_time / timeout_s / last_submit_time / end_time / now / remaining` -## 工作区与目录 -**你的当前目录就是工作区。所有需要的文件都在当前目录下。** -- **禁止 `cd` 到当前目录之外**（不要访问父目录或其他路径） -- **只使用相对路径**（如 `./code/train.py`，而非绝对路径） -- 如果看到 symlink 指向外部路径，忽略它——直接用相对路径访问即可 +## Workspace and Directory Layout +**Your current directory is the workspace. All required files are located under the current directory.** +- **Do not `cd` outside the current directory**. Do not access parent directories or unrelated paths. +- **Use only relative paths** such as `./code/train.py`, not absolute paths. +- If you see a symlink pointing outside, ignore that fact and access it through the relative path here. -目录结构： -``` +Directory structure: +```text ./ -├── code/ # 你的代码区（所有自行编写的代码放在此处） -├── data/ # 训练数据（只读） -├── models/ # 基础模型（只读） -├── output/ # 模型输出（训练好的模型保存在此） -├── description.md # 任务描述（必读） -├── instructions.md # 本文件 -├── run_meta.json # 时间与预算信号（单一事实源） -└── ... # benchmark 特有文件（用 ls 查看完整列表） +├── code/ # Your code area (put all self-written code here) +├── data/ # Training data (read-only) +├── models/ # Base model (read-only) +├── output/ # Model outputs (save trained models here) +├── description.md # Task description (required reading) +├── instructions.md # This file +├── run_meta.json # Time and budget signals (single source of truth) +└── ... # Benchmark-specific files (use ls to see the full list) ``` -**先 `ls` 查看当前目录所有可用文件。** 不同类型的 benchmark 会提供不同的额外文件： -- **交互式环境类**（如 ALFWorld）：会提供 `eval.py`（环境交互 + 评测逻辑）、prompt 模板、配置文件等——这些是编写训练代码的关键参考 -- **静态数据集类**（如 GSM8K）：主要通过 `data/` 下的数据文件提供训练样本 +**Run `ls` first to inspect all available files in the current directory.** Different benchmark types may provide different extra files: +- **Interactive environments** (such as ALFWorld): may provide `eval.py` (environment interaction plus evaluation logic), prompt templates, config files, and similar artifacts. These are critical references for writing training code. +- **Static datasets** (such as GSM8K): usually expose training samples mainly through files under `data/`. -**说明**： -- `code/`：在此编写代码（文件名和结构自由组织） -- `output/`：训练产出的模型存放处。可存放多个版本（如 `output/v1/`、`output/v2/`），提交时指定具体路径 +**Notes**: +- `code/`: write your code here. You may organize filenames and structure freely. +- `output/`: store trained model artifacts here. You may keep multiple versions such as `output/v1/` and `output/v2/`, and specify the exact path at submission time. -## 任务流程（固定时间内刷分） -1. 探索工作区，阅读 `description.md`、`instructions.md` 和相关文件（如有 `eval.py` 务必仔细阅读），了解任务目标和可用资源 -2. 在 `code/` 下编写代码，训练模型（SFT、GRPO、PPO 等均可） -3. 保存模型到 $OUTPUT_DIR（如 `output/v1`） -4. 提交评测：POST $GRADING_SERVER_URL/submit -5. 根据返回的 score 调整策略，**在剩余时间内持续迭代并提交更高分模型** +## Task Loop (Improve Score Within the Budget) +1. Explore the workspace. Read `description.md`, `instructions.md`, and other relevant files. If `eval.py` is present, read it carefully. +2. Write code under `code/` and train a model. SFT, GRPO, PPO, and other methods are all allowed. +3. Save the resulting model to `$OUTPUT_DIR` such as `output/v1`. +4. Submit it for evaluation through `POST $GRADING_SERVER_URL/submit`. +5. Adjust your strategy based on the returned score and **keep iterating toward a better model within the remaining time**. ## API ```bash -# 提交评测（指定模型路径，返回 score、improvement、best） +# Submit a model for evaluation (returns score, improvement, and best) curl -X POST "$GRADING_SERVER_URL/submit" \ -H "Content-Type: application/json" \ -d '{"model_path": "'$OUTPUT_DIR'/v1"}' -# 指定 GPU 评测（可选，默认使用 GPU 0） +# Evaluate on a specific GPU (optional; GPU 0 is used by default) curl -X POST "$GRADING_SERVER_URL/submit" \ -H "Content-Type: application/json" \ -d '{"model_path": "'$OUTPUT_DIR'/v1", "gpu": "0"}' -# 多卡评测 +# Multi-GPU evaluation curl -X POST "$GRADING_SERVER_URL/submit" \ -H "Content-Type: application/json" \ -d '{"model_path": "'$OUTPUT_DIR'/v1", "gpu": "2,3"}' -# 查询时间与预算（优先使用 run_meta.json；此 API 仅做补充） +# Query time and budget (prefer run_meta.json; this API is supplementary) curl "$GRADING_SERVER_URL/time" -# 健康检查（返回可用 GPU 列表等信息） +# Health check (returns available GPU list and related status) curl "$GRADING_SERVER_URL/health" ``` -### /submit 参数 -| 参数 | 类型 | 必填 | 说明 | +### `/submit` Parameters +| Parameter | Type | Required | Description | |------|------|------|------| -| model_path | string | 是 | 模型路径 | -| gpu | string | 否 | 指定 GPU（如 "0"、"1"、"0,1"），必须是可用 GPU 之一。不传则默认使用第一个可用 GPU。可通过 /health 查看可用列表 | +| model_path | string | Yes | Model path | +| gpu | string | No | Requested GPU(s), such as `"0"`, `"1"`, or `"0,1"`. Must be chosen from the available GPU list. If omitted, the first available GPU is used by default. You can inspect the available list through `/health`. | -### /submit 响应示例 +### `/submit` Response Example ```json { "submission_id": 3, @@ -106,9 +106,9 @@ curl "$GRADING_SERVER_URL/health" } ``` -## 注意 -- **不要直接复制/symlink 基座模型提交**——未经训练的基座模型只会得到 baseline 分数（improvement = 0），这是浪费提交机会。必须先训练再提交。 -- 可多次提交不同版本的模型，系统自动跟踪最高分 -- 合理利用时间，根据 score 反馈迭代优化 -- **必须提交完整模型**：评测系统不支持 LoRA adapter 目录。如果用 LoRA/PEFT 训练，保存前必须合并：`model = model.merge_and_unload(); model.save_pretrained(output_path); tokenizer.save_pretrained(output_path)` -- trl 保存模型后，`tokenizer_config.json` 中的 `extra_special_tokens` 会被保存为 list 格式，但 vLLM/transformers 加载时需要 dict 格式。保存模型后需删除该字段，否则评测会失败。 +## Important Notes +- **Do not directly submit a copied or symlinked base model**. An untrained base model only receives the baseline score (`improvement = 0`), which wastes a submission. You must train before submitting. +- You may submit multiple model versions. The system automatically tracks the best score. +- Use the remaining time carefully and iterate based on score feedback. +- **You must submit a full model**. The evaluation system does not support a LoRA adapter directory alone. If you train with LoRA/PEFT, merge before saving: `model = model.merge_and_unload(); model.save_pretrained(output_path); tokenizer.save_pretrained(output_path)`. +- After saving a model with `trl`, the `extra_special_tokens` field in `tokenizer_config.json` may be stored as a list, while vLLM/transformers expects a dict during loading. Remove that field after saving, or evaluation may fail.