Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Manuel Raimann
|
28b1687be6 | ||
|
Manuel Raimann
|
fc0559a02d | ||
|
Manuel Raimann
|
6fb39d17c1 |
+2
-150
@@ -6,10 +6,6 @@ on:
|
||||
pull_request:
|
||||
branches: [main, master]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
@@ -62,27 +58,6 @@ jobs:
|
||||
cargo test --verbose --features "${{ matrix.features }}"
|
||||
fi
|
||||
|
||||
examples:
|
||||
name: Examples
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Install Rust
|
||||
run: |
|
||||
rustup override set stable
|
||||
rustup update stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- name: Run sync example (ml_hyperparameter_tuning)
|
||||
run: cargo run --example ml_hyperparameter_tuning
|
||||
- name: Run sync example (benchmark_convergence)
|
||||
run: cargo run --example benchmark_convergence
|
||||
- name: Run derive example (parameter_api)
|
||||
run: cargo run --example parameter_api --features derive
|
||||
- name: Run async example (async_api_optimization)
|
||||
run: cargo run --example async_api_optimization --features async
|
||||
- name: Run visualization example (visualization_demo)
|
||||
run: cargo run --example visualization_demo --features visualization
|
||||
|
||||
docs:
|
||||
name: Docs
|
||||
runs-on: ubuntu-latest
|
||||
@@ -220,27 +195,6 @@ jobs:
|
||||
- name: Run tests
|
||||
run: cargo test --all-features
|
||||
|
||||
cross-targets:
|
||||
name: ${{ matrix.target }}
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
target:
|
||||
- aarch64-unknown-linux-gnu
|
||||
- i686-unknown-linux-gnu
|
||||
- powerpc64le-unknown-linux-gnu
|
||||
- s390x-unknown-linux-gnu
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Install Rust
|
||||
run: |
|
||||
rustup override set stable
|
||||
rustup update stable
|
||||
rustup target add ${{ matrix.target }}
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- name: Check compilation
|
||||
run: cargo check --all-features --target ${{ matrix.target }}
|
||||
|
||||
typos:
|
||||
name: Typos
|
||||
@@ -257,99 +211,6 @@ jobs:
|
||||
- name: Typos Check
|
||||
run: typos src/
|
||||
|
||||
bench-check:
|
||||
name: Benchmark Smoke Test
|
||||
if: github.event_name == 'push'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Install Rust
|
||||
run: |
|
||||
rustup override set stable
|
||||
rustup update stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- name: Run benchmarks (smoke test)
|
||||
run: cargo bench --all-features --bench samplers --bench optimization
|
||||
|
||||
bench-compare:
|
||||
name: Benchmark Comparison
|
||||
if: github.event_name == 'pull_request'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Install Rust
|
||||
run: |
|
||||
rustup override set stable
|
||||
rustup update stable
|
||||
|
||||
- name: Install critcmp
|
||||
run: cargo install critcmp
|
||||
|
||||
# --- Run benchmarks on the BASE (target) branch ---
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.base.sha }}
|
||||
clean: false
|
||||
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
key: bench-base
|
||||
|
||||
- name: Bench baseline
|
||||
run: cargo bench --all-features --bench samplers --bench optimization -- --save-baseline base
|
||||
|
||||
# --- Run benchmarks on the HEAD (PR) branch ---
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.sha }}
|
||||
clean: false
|
||||
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
key: bench-head
|
||||
|
||||
- name: Bench PR head
|
||||
run: cargo bench --all-features --bench samplers --bench optimization -- --save-baseline head
|
||||
|
||||
# --- Compare and post comment ---
|
||||
- name: Compare benchmarks
|
||||
id: compare
|
||||
run: |
|
||||
EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
|
||||
echo "result<<$EOF" >> "$GITHUB_OUTPUT"
|
||||
critcmp base head --color never >> "$GITHUB_OUTPUT"
|
||||
echo "$EOF" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Find existing comment
|
||||
uses: peter-evans/find-comment@v4
|
||||
id: find-comment
|
||||
with:
|
||||
issue-number: ${{ github.event.pull_request.number }}
|
||||
comment-author: github-actions[bot]
|
||||
body-includes: "## Benchmark Comparison"
|
||||
|
||||
- name: Post or update PR comment
|
||||
uses: peter-evans/create-or-update-comment@v5
|
||||
with:
|
||||
comment-id: ${{ steps.find-comment.outputs.comment-id }}
|
||||
issue-number: ${{ github.event.pull_request.number }}
|
||||
edit-mode: replace
|
||||
body: |
|
||||
## Benchmark Comparison
|
||||
|
||||
**Base:** `${{ github.event.pull_request.base.sha }}` | **Head:** `${{ github.event.pull_request.head.sha }}`
|
||||
|
||||
<details>
|
||||
<summary>Click to expand full results</summary>
|
||||
|
||||
```
|
||||
${{ steps.compare.outputs.result }}
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
> Benchmarks run on `ubuntu-latest` via [Criterion](https://github.com/bheisler/criterion.rs) + [critcmp](https://github.com/BurntSushi/critcmp).
|
||||
> Results may vary due to shared CI runners. Look for consistent >5% changes.
|
||||
|
||||
publish:
|
||||
name: Publish to crates.io
|
||||
runs-on: ubuntu-latest
|
||||
@@ -358,7 +219,6 @@ jobs:
|
||||
- fmt
|
||||
- clippy
|
||||
- test
|
||||
- examples
|
||||
- docs
|
||||
- feature-check
|
||||
- coverage
|
||||
@@ -368,7 +228,6 @@ jobs:
|
||||
- semver
|
||||
- minimal-versions
|
||||
- cross-platform
|
||||
- cross-targets
|
||||
- typos
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -394,14 +253,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Publish
|
||||
if: steps.check.outputs.skip == 'false'
|
||||
run: |
|
||||
cargo publish --allow-dirty 2>&1 | tee publish_output.txt || {
|
||||
if grep -q "already uploaded" publish_output.txt; then
|
||||
echo "Version already published, skipping"
|
||||
exit 0
|
||||
fi
|
||||
exit 1
|
||||
}
|
||||
if: steps.check.outputs.skip == 'false' && github.ref == 'refs/heads/master'
|
||||
run: cargo publish
|
||||
env:
|
||||
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
|
||||
|
||||
@@ -5,9 +5,6 @@ on:
|
||||
- cron: "0 6 * * *" # Daily at 6:00 UTC
|
||||
workflow_dispatch: # Allow manual trigger
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
|
||||
+4
-53
@@ -1,72 +1,23 @@
|
||||
[workspace]
|
||||
members = ["optimizer-derive"]
|
||||
|
||||
[package]
|
||||
name = "optimizer"
|
||||
version = "0.8.0"
|
||||
version = "0.2.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.88"
|
||||
license = "MIT"
|
||||
authors = ["Manuel Raimann <raimannma@outlook.de"]
|
||||
description = "A Rust library for optimization algorithms."
|
||||
repository = "https://github.com/raimannma/rust-optimizer"
|
||||
documentation = "https://docs.rs/optimizer"
|
||||
keywords = ["optimization", "hyperparameter", "tpe", "grid-search", "bayesian"]
|
||||
categories = ["algorithm", "science", "data-structures"]
|
||||
readme = "README.md"
|
||||
|
||||
|
||||
[dependencies]
|
||||
rand = "0.10"
|
||||
rand = "0.9"
|
||||
thiserror = "2"
|
||||
parking_lot = "0.12"
|
||||
tokio = { version = "1", features = ["sync", "rt-multi-thread"], optional = true }
|
||||
optimizer-derive = { version = "0.1.0", path = "optimizer-derive", optional = true }
|
||||
serde = { version = "1", features = ["derive"], optional = true }
|
||||
serde_json = { version = "1", optional = true }
|
||||
tracing = { version = "0.1.29", optional = true }
|
||||
sobol_burley = { version = "0.5", optional = true }
|
||||
nalgebra = { version = "0.34", optional = true }
|
||||
|
||||
[features]
|
||||
default = []
|
||||
async = ["dep:tokio"]
|
||||
derive = ["dep:optimizer-derive"]
|
||||
serde = ["dep:serde", "dep:serde_json"]
|
||||
tracing = ["dep:tracing"]
|
||||
sobol = ["dep:sobol_burley"]
|
||||
cma-es = ["dep:nalgebra"]
|
||||
visualization = []
|
||||
fanova = []
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] }
|
||||
optimizer-derive = { version = "0.1.0", path = "optimizer-derive" }
|
||||
serde_json = "1"
|
||||
criterion = { version = "0.8", features = ["html_reports"] }
|
||||
|
||||
[[bench]]
|
||||
name = "samplers"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "optimization"
|
||||
harness = false
|
||||
|
||||
[[example]]
|
||||
name = "async_api_optimization"
|
||||
path = "examples/async_api_optimization.rs"
|
||||
required-features = ["async"]
|
||||
|
||||
[[example]]
|
||||
name = "ml_hyperparameter_tuning"
|
||||
path = "examples/ml_hyperparameter_tuning.rs"
|
||||
|
||||
[[example]]
|
||||
name = "parameter_api"
|
||||
path = "examples/parameter_api.rs"
|
||||
required-features = ["derive"]
|
||||
|
||||
[[example]]
|
||||
name = "visualization_demo"
|
||||
path = "examples/visualization_demo.rs"
|
||||
required-features = ["visualization"]
|
||||
tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# optimizer
|
||||
|
||||
A Rust library for black-box optimization with multiple sampling strategies.
|
||||
A Rust library for black-box optimization using Tree-Parzen Estimator (TPE).
|
||||
|
||||
[](https://docs.rs/optimizer)
|
||||
[](https://crates.io/crates/optimizer)
|
||||
@@ -9,143 +9,32 @@ A Rust library for black-box optimization with multiple sampling strategies.
|
||||
## Features
|
||||
|
||||
- Optuna-like API for hyperparameter optimization
|
||||
- Multiple sampling strategies:
|
||||
- **Random Search** - Simple random sampling for baseline comparisons
|
||||
- **TPE (Tree-Parzen Estimator)** - Bayesian optimization for efficient search
|
||||
- **Grid Search** - Exhaustive search over a specified parameter grid
|
||||
- Float, integer, categorical, boolean, and enum parameter types
|
||||
- Float, integer, and categorical parameter types
|
||||
- Log-scale and stepped parameter sampling
|
||||
- Sync and async optimization with parallel trial evaluation
|
||||
- `#[derive(Categorical)]` for enum parameters
|
||||
|
||||
## Quick Start
|
||||
|
||||
```rust
|
||||
use optimizer::parameter::{FloatParam, Parameter};
|
||||
use optimizer::sampler::tpe::TpeSampler;
|
||||
use optimizer::{Direction, Study};
|
||||
use optimizer::{Direction, Study, TpeSampler};
|
||||
|
||||
// Create a study with TPE sampler
|
||||
let sampler = TpeSampler::builder().seed(42).build().unwrap();
|
||||
let sampler = TpeSampler::builder().seed(42).build();
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
// Define parameter search space
|
||||
let x_param = FloatParam::new(-10.0, 10.0);
|
||||
|
||||
// Optimize x^2 for 20 trials
|
||||
study
|
||||
.optimize_with_sampler(20, |trial| {
|
||||
let x = x_param.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(x * x)
|
||||
let x = trial.suggest_float("x", -10.0, 10.0)?;
|
||||
Ok::<_, optimizer::TpeError>(x * x)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// Get the best result
|
||||
let best = study.best_trial().unwrap();
|
||||
println!("Best value: {}", best.value);
|
||||
for (id, label) in &best.param_labels {
|
||||
println!(" {}: {:?}", label, best.params[id]);
|
||||
}
|
||||
```
|
||||
|
||||
## Samplers
|
||||
|
||||
### Random Search
|
||||
|
||||
```rust
|
||||
use optimizer::{Direction, Study};
|
||||
use optimizer::sampler::random::RandomSampler;
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(
|
||||
Direction::Minimize,
|
||||
RandomSampler::with_seed(42),
|
||||
);
|
||||
```
|
||||
|
||||
### TPE (Tree-Parzen Estimator)
|
||||
|
||||
```rust
|
||||
use optimizer::{Direction, Study};
|
||||
use optimizer::sampler::tpe::TpeSampler;
|
||||
|
||||
let sampler = TpeSampler::builder()
|
||||
.gamma(0.15) // Quantile for good/bad split
|
||||
.n_startup_trials(20) // Random trials before TPE kicks in
|
||||
.n_ei_candidates(32) // Candidates to evaluate
|
||||
.seed(42)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
```
|
||||
|
||||
#### Gamma Strategies
|
||||
|
||||
The gamma parameter controls what fraction of trials are considered "good" when building the TPE model. Instead of a fixed value, you can use adaptive strategies:
|
||||
|
||||
| Strategy | Description | Formula |
|
||||
|----------|-------------|---------|
|
||||
| `FixedGamma` | Constant value (default: 0.25) | `γ = constant` |
|
||||
| `LinearGamma` | Linear interpolation over trials | `γ = γ_min + (γ_max - γ_min) * min(n/n_max, 1)` |
|
||||
| `SqrtGamma` | Optuna-style inverse sqrt scaling | `γ = min(γ_max, factor/√n / n)` |
|
||||
| `HyperoptGamma` | Hyperopt-style adaptive | `γ = min(γ_max, (base + 1) / n)` |
|
||||
|
||||
```rust
|
||||
use optimizer::sampler::tpe::{TpeSampler, SqrtGamma, LinearGamma};
|
||||
|
||||
// Optuna-style gamma that decreases with more trials
|
||||
let sampler = TpeSampler::builder()
|
||||
.gamma_strategy(SqrtGamma::default())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Linear interpolation from 0.1 to 0.3 over 100 trials
|
||||
let sampler = TpeSampler::builder()
|
||||
.gamma_strategy(LinearGamma::new(0.1, 0.3, 100).unwrap())
|
||||
.build()
|
||||
.unwrap();
|
||||
```
|
||||
|
||||
You can also implement custom strategies:
|
||||
|
||||
```rust
|
||||
use optimizer::sampler::tpe::{TpeSampler, GammaStrategy};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct MyGamma { base: f64 }
|
||||
|
||||
impl GammaStrategy for MyGamma {
|
||||
fn gamma(&self, n_trials: usize) -> f64 {
|
||||
(self.base + 0.01 * n_trials as f64).min(0.5)
|
||||
}
|
||||
fn clone_box(&self) -> Box<dyn GammaStrategy> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
}
|
||||
|
||||
let sampler = TpeSampler::builder()
|
||||
.gamma_strategy(MyGamma { base: 0.1 })
|
||||
.build()
|
||||
.unwrap();
|
||||
```
|
||||
|
||||
### Grid Search
|
||||
|
||||
```rust
|
||||
use optimizer::{Direction, Study};
|
||||
use optimizer::sampler::grid::GridSearchSampler;
|
||||
|
||||
let sampler = GridSearchSampler::builder()
|
||||
.n_points_per_param(10) // Number of points per parameter dimension
|
||||
.build();
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
println!("Best value: {} at x={:?}", best.value, best.params);
|
||||
```
|
||||
|
||||
## Feature Flags
|
||||
|
||||
- `async` - Enable async optimization methods (requires tokio)
|
||||
- `derive` - Enable `#[derive(Categorical)]` for enum parameters
|
||||
|
||||
## Documentation
|
||||
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
#[allow(dead_code)]
|
||||
mod test_functions;
|
||||
|
||||
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
|
||||
use optimizer::parameter::Parameter;
|
||||
use optimizer::sampler::random::RandomSampler;
|
||||
use optimizer::sampler::tpe::TpeSampler;
|
||||
use optimizer::{FloatParam, Study};
|
||||
|
||||
fn make_params(dims: usize) -> Vec<FloatParam> {
|
||||
(0..dims)
|
||||
.map(|i| FloatParam::new(-5.0, 5.0).name(format!("x{i}")))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn bench_tpe_sphere(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("tpe_sphere");
|
||||
group.sample_size(10);
|
||||
|
||||
for dims in [2, 10, 50] {
|
||||
let params = make_params(dims);
|
||||
group.bench_with_input(BenchmarkId::new("dims", dims), ¶ms, |b, params| {
|
||||
b.iter(|| {
|
||||
let study = Study::minimize(TpeSampler::builder().seed(42).build().unwrap());
|
||||
study
|
||||
.optimize(100, |trial| {
|
||||
let x: Vec<f64> = params
|
||||
.iter()
|
||||
.map(|p| p.suggest(trial))
|
||||
.collect::<Result<_, _>>()
|
||||
.unwrap();
|
||||
Ok::<_, optimizer::Error>(test_functions::sphere(&x))
|
||||
})
|
||||
.unwrap();
|
||||
});
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_tpe_rosenbrock(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("tpe_rosenbrock");
|
||||
group.sample_size(10);
|
||||
|
||||
for dims in [2, 10] {
|
||||
let params = make_params(dims);
|
||||
group.bench_with_input(BenchmarkId::new("dims", dims), ¶ms, |b, params| {
|
||||
b.iter(|| {
|
||||
let study = Study::minimize(TpeSampler::builder().seed(42).build().unwrap());
|
||||
study
|
||||
.optimize(100, |trial| {
|
||||
let x: Vec<f64> = params
|
||||
.iter()
|
||||
.map(|p| p.suggest(trial))
|
||||
.collect::<Result<_, _>>()
|
||||
.unwrap();
|
||||
Ok::<_, optimizer::Error>(test_functions::rosenbrock(&x))
|
||||
})
|
||||
.unwrap();
|
||||
});
|
||||
});
|
||||
}
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_random_vs_tpe(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("random_vs_tpe");
|
||||
group.sample_size(10);
|
||||
let params = make_params(5);
|
||||
|
||||
group.bench_function("random_5d", |b| {
|
||||
b.iter(|| {
|
||||
let study = Study::minimize(RandomSampler::with_seed(42));
|
||||
study
|
||||
.optimize(100, |trial| {
|
||||
let x: Vec<f64> = params
|
||||
.iter()
|
||||
.map(|p| p.suggest(trial))
|
||||
.collect::<Result<_, _>>()
|
||||
.unwrap();
|
||||
Ok::<_, optimizer::Error>(test_functions::sphere(&x))
|
||||
})
|
||||
.unwrap();
|
||||
});
|
||||
});
|
||||
|
||||
group.bench_function("tpe_5d", |b| {
|
||||
b.iter(|| {
|
||||
let study = Study::minimize(TpeSampler::builder().seed(42).build().unwrap());
|
||||
study
|
||||
.optimize(100, |trial| {
|
||||
let x: Vec<f64> = params
|
||||
.iter()
|
||||
.map(|p| p.suggest(trial))
|
||||
.collect::<Result<_, _>>()
|
||||
.unwrap();
|
||||
Ok::<_, optimizer::Error>(test_functions::sphere(&x))
|
||||
})
|
||||
.unwrap();
|
||||
});
|
||||
});
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(
|
||||
benches,
|
||||
bench_tpe_sphere,
|
||||
bench_tpe_rosenbrock,
|
||||
bench_random_vs_tpe
|
||||
);
|
||||
criterion_main!(benches);
|
||||
@@ -1,118 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
|
||||
use optimizer::parameter::Parameter;
|
||||
use optimizer::sampler::Sampler;
|
||||
use optimizer::sampler::grid::GridSearchSampler;
|
||||
use optimizer::sampler::random::RandomSampler;
|
||||
use optimizer::sampler::tpe::TpeSampler;
|
||||
use optimizer::{CompletedTrial, FloatParam};
|
||||
|
||||
/// Build a synthetic history of `n` completed trials over `dims` float parameters.
|
||||
fn build_history(n: usize, dims: usize) -> Vec<CompletedTrial<f64>> {
|
||||
let params: Vec<FloatParam> = (0..dims)
|
||||
.map(|i| FloatParam::new(-5.0, 5.0).name(format!("x{i}")))
|
||||
.collect();
|
||||
|
||||
let mut history = Vec::with_capacity(n);
|
||||
let sampler = RandomSampler::with_seed(42);
|
||||
|
||||
for trial_id in 0..n {
|
||||
let id = trial_id as u64;
|
||||
let mut param_values = HashMap::new();
|
||||
let mut distributions = HashMap::new();
|
||||
let mut param_labels = HashMap::new();
|
||||
for p in ¶ms {
|
||||
let dist = p.distribution();
|
||||
let val = sampler.sample(&dist, id, &history);
|
||||
param_values.insert(p.id(), val);
|
||||
distributions.insert(p.id(), dist);
|
||||
param_labels.insert(p.id(), p.label());
|
||||
}
|
||||
// Use sphere function value as objective
|
||||
let value: f64 = param_values
|
||||
.values()
|
||||
.map(|v| {
|
||||
let optimizer::ParamValue::Float(f) = v else {
|
||||
unreachable!()
|
||||
};
|
||||
f * f
|
||||
})
|
||||
.sum();
|
||||
history.push(CompletedTrial::new(
|
||||
id,
|
||||
param_values,
|
||||
distributions,
|
||||
param_labels,
|
||||
value,
|
||||
));
|
||||
}
|
||||
history
|
||||
}
|
||||
|
||||
fn bench_tpe_sample(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("tpe_sample");
|
||||
let dist = FloatParam::new(-5.0, 5.0).distribution();
|
||||
let tpe = TpeSampler::builder().seed(42).build().unwrap();
|
||||
|
||||
for history_size in [10, 100, 1000] {
|
||||
let history = build_history(history_size, 2);
|
||||
group.bench_with_input(
|
||||
BenchmarkId::new("history", history_size),
|
||||
&history,
|
||||
|b, history| {
|
||||
b.iter(|| tpe.sample(&dist, history.len() as u64, history));
|
||||
},
|
||||
);
|
||||
}
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_random_sample(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("random_sample");
|
||||
let dist = FloatParam::new(-5.0, 5.0).distribution();
|
||||
let sampler = RandomSampler::with_seed(42);
|
||||
|
||||
for history_size in [10, 100, 1000] {
|
||||
let history = build_history(history_size, 2);
|
||||
group.bench_with_input(
|
||||
BenchmarkId::new("history", history_size),
|
||||
&history,
|
||||
|b, history| {
|
||||
b.iter(|| sampler.sample(&dist, history.len() as u64, history));
|
||||
},
|
||||
);
|
||||
}
|
||||
group.finish();
|
||||
}
|
||||
|
||||
fn bench_grid_sample(c: &mut Criterion) {
|
||||
let mut group = c.benchmark_group("grid_sample");
|
||||
let dist = FloatParam::new(-5.0, 5.0).distribution();
|
||||
let history: Vec<CompletedTrial<f64>> = Vec::new();
|
||||
|
||||
for grid_points in [5, 10, 50] {
|
||||
group.bench_with_input(
|
||||
BenchmarkId::new("points", grid_points),
|
||||
&grid_points,
|
||||
|b, _| {
|
||||
b.iter(|| {
|
||||
// Fresh sampler each iteration since grid tracks used points
|
||||
let sampler = GridSearchSampler::builder()
|
||||
.n_points_per_param(grid_points)
|
||||
.build();
|
||||
sampler.sample(&dist, 0, &history)
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(
|
||||
benches,
|
||||
bench_tpe_sample,
|
||||
bench_random_sample,
|
||||
bench_grid_sample
|
||||
);
|
||||
criterion_main!(benches);
|
||||
@@ -1,84 +0,0 @@
|
||||
//! Standard optimization test functions for benchmarking.
|
||||
|
||||
/// Sphere function: unimodal, convex. Global minimum f(0,...,0) = 0.
|
||||
pub fn sphere(x: &[f64]) -> f64 {
|
||||
x.iter().map(|xi| xi * xi).sum()
|
||||
}
|
||||
|
||||
/// Rosenbrock function: narrow valley. Global minimum f(1,...,1) = 0.
|
||||
pub fn rosenbrock(x: &[f64]) -> f64 {
|
||||
x.windows(2)
|
||||
.map(|w| 100.0 * (w[1] - w[0] * w[0]).powi(2) + (1.0 - w[0]).powi(2))
|
||||
.sum()
|
||||
}
|
||||
|
||||
/// Rastrigin function: highly multimodal. Global minimum f(0,...,0) = 0.
|
||||
pub fn rastrigin(x: &[f64]) -> f64 {
|
||||
let n = x.len() as f64;
|
||||
10.0 * n
|
||||
+ x.iter()
|
||||
.map(|xi| xi * xi - 10.0 * (2.0 * std::f64::consts::PI * xi).cos())
|
||||
.sum::<f64>()
|
||||
}
|
||||
|
||||
/// Ackley function: nearly flat with a deep well. Global minimum f(0,...,0) = 0.
|
||||
pub fn ackley(x: &[f64]) -> f64 {
|
||||
let n = x.len() as f64;
|
||||
let sum_sq: f64 = x.iter().map(|xi| xi * xi).sum();
|
||||
let sum_cos: f64 = x
|
||||
.iter()
|
||||
.map(|xi| (2.0 * std::f64::consts::PI * xi).cos())
|
||||
.sum();
|
||||
-20.0 * (-0.2 * (sum_sq / n).sqrt()).exp() - (sum_cos / n).exp() + 20.0 + std::f64::consts::E
|
||||
}
|
||||
|
||||
/// Branin function (2D only). Three global minima with f* ≈ 0.397887.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `x` does not have exactly 2 elements.
|
||||
pub fn branin(x: &[f64]) -> f64 {
|
||||
assert!(x.len() == 2, "Branin requires exactly 2 dimensions");
|
||||
let (x1, x2) = (x[0], x[1]);
|
||||
let pi = std::f64::consts::PI;
|
||||
let a = 1.0;
|
||||
let b = 5.1 / (4.0 * pi * pi);
|
||||
let c = 5.0 / pi;
|
||||
let r = 6.0;
|
||||
let s = 10.0;
|
||||
let t = 1.0 / (8.0 * pi);
|
||||
a * (x2 - b * x1 * x1 + c * x1 - r).powi(2) + s * (1.0 - t) * x1.cos() + s
|
||||
}
|
||||
|
||||
/// Hartmann 6D function. Global minimum f* ≈ -3.3224.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `x` does not have exactly 6 elements.
|
||||
pub fn hartmann6(x: &[f64]) -> f64 {
|
||||
assert!(x.len() == 6, "Hartmann6 requires exactly 6 dimensions");
|
||||
|
||||
let alpha = [1.0, 1.2, 3.0, 3.2];
|
||||
let a_matrix = [
|
||||
[10.0, 3.0, 17.0, 3.5, 1.7, 8.0],
|
||||
[0.05, 10.0, 17.0, 0.1, 8.0, 14.0],
|
||||
[3.0, 3.5, 1.7, 10.0, 17.0, 8.0],
|
||||
[17.0, 8.0, 0.05, 10.0, 0.1, 14.0],
|
||||
];
|
||||
let p_matrix = [
|
||||
[0.1312, 0.1696, 0.5569, 0.0124, 0.8283, 0.5886],
|
||||
[0.2329, 0.4135, 0.8307, 0.3736, 0.1004, 0.9991],
|
||||
[0.2348, 0.1451, 0.3522, 0.2883, 0.3047, 0.6650],
|
||||
[0.4047, 0.8828, 0.8732, 0.5743, 0.1091, 0.0381],
|
||||
];
|
||||
|
||||
let mut result = 0.0;
|
||||
for i in 0..4 {
|
||||
let mut inner = 0.0;
|
||||
for (j, xj) in x.iter().enumerate() {
|
||||
inner += a_matrix[i][j] * (xj - p_matrix[i][j]).powi(2);
|
||||
}
|
||||
result -= alpha[i] * (-inner).exp();
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -1,11 +1,7 @@
|
||||
[advisories]
|
||||
version = 2
|
||||
db-path = "~/.cargo/advisory-db"
|
||||
ignore = [
|
||||
# paste is unmaintained but it's a transitive dep via simba -> nalgebra
|
||||
# with no fix available upstream yet
|
||||
"RUSTSEC-2024-0436",
|
||||
]
|
||||
ignore = []
|
||||
|
||||
[licenses]
|
||||
version = 2
|
||||
@@ -13,6 +9,7 @@ allow = [
|
||||
"MIT",
|
||||
"Apache-2.0",
|
||||
"Apache-2.0 WITH LLVM-exception",
|
||||
"BSD-2-Clause",
|
||||
"Unicode-3.0",
|
||||
]
|
||||
confidence-threshold = 0.8
|
||||
|
||||
@@ -1,368 +0,0 @@
|
||||
//! Async API Parameter Optimization Example
|
||||
//!
|
||||
//! This example shows how to use async/parallel optimization to tune
|
||||
//! configuration parameters for a web service. Each evaluation simulates
|
||||
//! an async operation (like deploying and load-testing a service).
|
||||
//!
|
||||
//! # Key Concepts Demonstrated
|
||||
//!
|
||||
//! - Async optimization with `optimize_parallel`
|
||||
//! - Running multiple trials concurrently for faster optimization
|
||||
//! - Boolean and categorical parameter types
|
||||
//! - Measuring speedup from parallelism
|
||||
//!
|
||||
//! # When to Use Async Optimization
|
||||
//!
|
||||
//! Use async/parallel optimization when your objective function involves:
|
||||
//! - Network requests (API calls, database queries)
|
||||
//! - File I/O operations
|
||||
//! - External service calls
|
||||
//! - Any operation where you're waiting for I/O rather than computing
|
||||
//!
|
||||
//! With parallelism, you can evaluate multiple configurations simultaneously,
|
||||
//! significantly reducing total optimization time.
|
||||
//!
|
||||
//! Run with: `cargo run --example async_api_optimization --features async`
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use optimizer::prelude::*;
|
||||
|
||||
// ============================================================================
|
||||
// Configuration: Service parameters we want to tune
|
||||
// ============================================================================
|
||||
|
||||
/// Configuration for a web service.
|
||||
///
|
||||
/// In a real application, these parameters would control:
|
||||
/// - Memory allocation (cache sizes)
|
||||
/// - Connection management (pool sizes, timeouts)
|
||||
/// - Request handling (batching, compression)
|
||||
/// - Protocol options (HTTP version, load balancing)
|
||||
struct ServiceConfig {
|
||||
cache_size_mb: i64,
|
||||
connection_pool_size: i64,
|
||||
request_timeout_ms: i64,
|
||||
retry_count: i64,
|
||||
batch_size: i64,
|
||||
compression_level: i64,
|
||||
use_http2: bool,
|
||||
load_balancing: String,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Objective Function: Evaluate a service configuration
|
||||
// ============================================================================
|
||||
|
||||
/// Simulates deploying and load-testing a service configuration.
|
||||
///
|
||||
/// In a real scenario, this function might:
|
||||
/// 1. Deploy the configuration to a staging environment
|
||||
/// 2. Run load tests against the service
|
||||
/// 3. Collect metrics (latency, throughput, error rate)
|
||||
/// 4. Return a composite score
|
||||
///
|
||||
/// The async sleep simulates the I/O time of these operations.
|
||||
/// This is where parallel execution helps - while one trial is waiting
|
||||
/// for I/O, other trials can run.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn evaluate_service(config: &ServiceConfig) -> f64 {
|
||||
// Simulate async I/O (deployment, load testing, metric collection)
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
|
||||
// Calculate a score based on how close we are to optimal values
|
||||
// Lower score = better configuration
|
||||
let mut score = 0.0;
|
||||
|
||||
// Cache size: too small = cache misses, too large = wasted memory
|
||||
// Optimal around 512MB
|
||||
let cache_optimal = 512.0;
|
||||
score += ((config.cache_size_mb as f64 - cache_optimal) / 256.0).powi(2);
|
||||
|
||||
// Connection pool: too small = contention, too large = resource waste
|
||||
// Optimal around 100
|
||||
let pool_optimal = 100.0;
|
||||
score += ((config.connection_pool_size as f64 - pool_optimal) / 50.0).powi(2);
|
||||
|
||||
// Timeout: too short = false failures, too long = slow recovery
|
||||
// Optimal around 5000ms
|
||||
let timeout_optimal = 5000.0;
|
||||
score += ((config.request_timeout_ms as f64 - timeout_optimal) / 2000.0).powi(2);
|
||||
|
||||
// Retries: too few = fragile, too many = amplifies failures
|
||||
// Optimal around 3
|
||||
let retry_optimal = 3.0;
|
||||
score += ((config.retry_count as f64 - retry_optimal) / 2.0).powi(2);
|
||||
|
||||
// Batch size: trade-off between latency and throughput
|
||||
// Optimal around 64
|
||||
let batch_optimal = 64.0;
|
||||
score += ((config.batch_size as f64 - batch_optimal) / 32.0).powi(2);
|
||||
|
||||
// Compression level: trade-off between CPU and bandwidth
|
||||
// Optimal around 6
|
||||
let compression_optimal = 6.0;
|
||||
score += ((config.compression_level as f64 - compression_optimal) / 3.0).powi(2);
|
||||
|
||||
// HTTP/2 is generally better for our use case
|
||||
if !config.use_http2 {
|
||||
score += 0.5;
|
||||
}
|
||||
|
||||
// Load balancing strategy affects performance
|
||||
score += match config.load_balancing.as_str() {
|
||||
"round_robin" => 0.0, // Best for our use case
|
||||
"least_connections" => 0.1, // Good alternative
|
||||
"ip_hash" => 0.2, // OK for session affinity
|
||||
"random" => 0.3, // Not ideal
|
||||
_ => 1.0,
|
||||
};
|
||||
|
||||
// Add noise to simulate real-world variability
|
||||
let noise = (config.cache_size_mb as f64 * 0.1).sin() * 0.05;
|
||||
|
||||
score + noise
|
||||
}
|
||||
|
||||
/// The async objective function for each trial.
|
||||
///
|
||||
/// For async optimization, the objective function must:
|
||||
/// 1. Take ownership of the Trial (not a mutable reference)
|
||||
/// 2. Return a Future
|
||||
/// 3. Return both the Trial and the result value as a tuple
|
||||
///
|
||||
/// This ownership pattern allows the trial to be used across await points.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn objective(
|
||||
mut trial: Trial,
|
||||
cache_size_mb_param: &IntParam,
|
||||
connection_pool_size_param: &IntParam,
|
||||
request_timeout_ms_param: &IntParam,
|
||||
retry_count_param: &IntParam,
|
||||
batch_size_param: &IntParam,
|
||||
compression_level_param: &IntParam,
|
||||
use_http2_param: &BoolParam,
|
||||
load_balancing_param: &CategoricalParam<&str>,
|
||||
) -> optimizer::Result<(Trial, f64)> {
|
||||
// Sample configuration parameters using parameter definitions
|
||||
let cache_size_mb = cache_size_mb_param.suggest(&mut trial)?;
|
||||
let connection_pool_size = connection_pool_size_param.suggest(&mut trial)?;
|
||||
let request_timeout_ms = request_timeout_ms_param.suggest(&mut trial)?;
|
||||
let retry_count = retry_count_param.suggest(&mut trial)?;
|
||||
let batch_size = batch_size_param.suggest(&mut trial)?;
|
||||
let compression_level = compression_level_param.suggest(&mut trial)?;
|
||||
let use_http2 = use_http2_param.suggest(&mut trial)?;
|
||||
let load_balancing = load_balancing_param.suggest(&mut trial)?;
|
||||
|
||||
// Build configuration
|
||||
let config = ServiceConfig {
|
||||
cache_size_mb,
|
||||
connection_pool_size,
|
||||
request_timeout_ms,
|
||||
retry_count,
|
||||
batch_size,
|
||||
compression_level,
|
||||
use_http2,
|
||||
load_balancing: load_balancing.to_string(),
|
||||
};
|
||||
|
||||
// Evaluate (this is the async part)
|
||||
let score = evaluate_service(&config).await;
|
||||
|
||||
// Return both the trial and the score
|
||||
Ok((trial, score))
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Helper Functions
|
||||
// ============================================================================
|
||||
|
||||
/// Prints the results of the optimization.
|
||||
fn print_results(study: &Study<f64>, elapsed: Duration, n_trials: usize) {
|
||||
println!("\n{}", "=".repeat(60));
|
||||
println!("\nOptimization completed!");
|
||||
println!("Total trials: {}", study.n_trials());
|
||||
println!("Time elapsed: {elapsed:.2?}");
|
||||
|
||||
// Calculate speedup from parallelism
|
||||
// Each trial takes ~50ms, so sequential would take n_trials * 50ms
|
||||
let sequential_time = n_trials as f64 * 0.050;
|
||||
let actual_time = elapsed.as_secs_f64();
|
||||
println!(
|
||||
"Effective parallelism: {:.1}x speedup",
|
||||
sequential_time / actual_time
|
||||
);
|
||||
}
|
||||
|
||||
/// Prints the best configuration found.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn print_best_config(
|
||||
study: &Study<f64>,
|
||||
cache_size_mb_param: &IntParam,
|
||||
connection_pool_size_param: &IntParam,
|
||||
request_timeout_ms_param: &IntParam,
|
||||
retry_count_param: &IntParam,
|
||||
batch_size_param: &IntParam,
|
||||
compression_level_param: &IntParam,
|
||||
use_http2_param: &BoolParam,
|
||||
load_balancing_param: &CategoricalParam<&str>,
|
||||
) -> optimizer::Result<()> {
|
||||
let best = study.best_trial()?;
|
||||
|
||||
println!("\nBest configuration found:");
|
||||
println!(" Score: {:.6}", best.value);
|
||||
println!("\n Parameters:");
|
||||
println!(
|
||||
" cache_size_mb: {}",
|
||||
best.get(cache_size_mb_param).unwrap()
|
||||
);
|
||||
println!(
|
||||
" connection_pool_size: {}",
|
||||
best.get(connection_pool_size_param).unwrap()
|
||||
);
|
||||
println!(
|
||||
" request_timeout_ms: {}",
|
||||
best.get(request_timeout_ms_param).unwrap()
|
||||
);
|
||||
println!(" retry_count: {}", best.get(retry_count_param).unwrap());
|
||||
println!(" batch_size: {}", best.get(batch_size_param).unwrap());
|
||||
println!(
|
||||
" compression_level: {}",
|
||||
best.get(compression_level_param).unwrap()
|
||||
);
|
||||
println!(" use_http2: {}", best.get(use_http2_param).unwrap());
|
||||
println!(
|
||||
" load_balancing: {}",
|
||||
best.get(load_balancing_param).unwrap()
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Prints the top N trials.
|
||||
fn print_top_trials(study: &Study<f64>, n: usize) {
|
||||
println!("\nTop {n} trials:");
|
||||
|
||||
let mut trials = study.trials();
|
||||
trials.sort_by(|a, b| a.value.partial_cmp(&b.value).unwrap());
|
||||
|
||||
for (i, trial) in trials.iter().take(n).enumerate() {
|
||||
println!(
|
||||
" {}. Trial #{}: score = {:.6}",
|
||||
i + 1,
|
||||
trial.id,
|
||||
trial.value
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Main: Set up and run the async optimization
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> optimizer::Result<()> {
|
||||
println!("=== Async API Parameter Optimization Example ===\n");
|
||||
|
||||
// Step 1: Create a TPE sampler
|
||||
let sampler = TpeSampler::builder()
|
||||
.n_startup_trials(8)
|
||||
.gamma(0.2)
|
||||
.seed(123)
|
||||
.build()
|
||||
.expect("Failed to build TPE sampler");
|
||||
|
||||
// Step 2: Create a study to minimize the score
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
// Step 3: Define parameter search spaces
|
||||
let cache_size_mb_param = IntParam::new(64, 1024).name("cache_size_mb").step(64);
|
||||
let connection_pool_size_param = IntParam::new(10, 200).name("connection_pool_size").step(10);
|
||||
let request_timeout_ms_param = IntParam::new(1000, 10000)
|
||||
.name("request_timeout_ms")
|
||||
.step(500);
|
||||
let retry_count_param = IntParam::new(0, 5).name("retry_count");
|
||||
let batch_size_param = IntParam::new(1, 256).name("batch_size").log_scale();
|
||||
let compression_level_param = IntParam::new(0, 9).name("compression_level");
|
||||
let use_http2_param = BoolParam::new().name("use_http2");
|
||||
let load_balancing_param = CategoricalParam::new(vec![
|
||||
"round_robin",
|
||||
"least_connections",
|
||||
"random",
|
||||
"ip_hash",
|
||||
])
|
||||
.name("load_balancing");
|
||||
|
||||
// Clone params for use after the closure moves them
|
||||
let cache_size_mb_p = cache_size_mb_param.clone();
|
||||
let connection_pool_size_p = connection_pool_size_param.clone();
|
||||
let request_timeout_ms_p = request_timeout_ms_param.clone();
|
||||
let retry_count_p = retry_count_param.clone();
|
||||
let batch_size_p = batch_size_param.clone();
|
||||
let compression_level_p = compression_level_param.clone();
|
||||
let use_http2_p = use_http2_param.clone();
|
||||
let load_balancing_p = load_balancing_param.clone();
|
||||
|
||||
// Step 4: Configure optimization
|
||||
let n_trials = 40;
|
||||
let concurrency = 4; // Run 4 trials in parallel
|
||||
|
||||
println!("Starting parallel optimization with {concurrency} concurrent evaluations...\n");
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
// Step 5: Run parallel async optimization
|
||||
//
|
||||
// optimize_parallel:
|
||||
// - Runs up to `concurrency` trials simultaneously
|
||||
// - Each trial calls the objective function
|
||||
// - Uses a semaphore to limit concurrent evaluations
|
||||
// - Collects results as trials complete
|
||||
//
|
||||
// The sampler gets access to trial history for informed sampling.
|
||||
study
|
||||
.optimize_parallel(n_trials, concurrency, move |trial| {
|
||||
let cache_size_mb_param = cache_size_mb_param.clone();
|
||||
let connection_pool_size_param = connection_pool_size_param.clone();
|
||||
let request_timeout_ms_param = request_timeout_ms_param.clone();
|
||||
let retry_count_param = retry_count_param.clone();
|
||||
let batch_size_param = batch_size_param.clone();
|
||||
let compression_level_param = compression_level_param.clone();
|
||||
let use_http2_param = use_http2_param.clone();
|
||||
let load_balancing_param = load_balancing_param.clone();
|
||||
async move {
|
||||
objective(
|
||||
trial,
|
||||
&cache_size_mb_param,
|
||||
&connection_pool_size_param,
|
||||
&request_timeout_ms_param,
|
||||
&retry_count_param,
|
||||
&batch_size_param,
|
||||
&compression_level_param,
|
||||
&use_http2_param,
|
||||
&load_balancing_param,
|
||||
)
|
||||
.await
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// Step 5: Print results
|
||||
print_results(&study, elapsed, n_trials);
|
||||
print_best_config(
|
||||
&study,
|
||||
&cache_size_mb_p,
|
||||
&connection_pool_size_p,
|
||||
&request_timeout_ms_p,
|
||||
&retry_count_p,
|
||||
&batch_size_p,
|
||||
&compression_level_p,
|
||||
&use_http2_p,
|
||||
&load_balancing_p,
|
||||
)?;
|
||||
print_top_trials(&study, 5);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,124 +0,0 @@
|
||||
use std::ops::ControlFlow;
|
||||
use std::time::Instant;
|
||||
|
||||
use optimizer::parameter::Parameter;
|
||||
use optimizer::sampler::random::RandomSampler;
|
||||
use optimizer::sampler::tpe::TpeSampler;
|
||||
use optimizer::{FloatParam, Study};
|
||||
|
||||
/// Standard optimization test functions.
|
||||
mod functions {
|
||||
pub fn sphere(x: &[f64]) -> f64 {
|
||||
x.iter().map(|xi| xi * xi).sum()
|
||||
}
|
||||
|
||||
pub fn rosenbrock(x: &[f64]) -> f64 {
|
||||
x.windows(2)
|
||||
.map(|w| 100.0 * (w[1] - w[0] * w[0]).powi(2) + (1.0 - w[0]).powi(2))
|
||||
.sum()
|
||||
}
|
||||
|
||||
pub fn rastrigin(x: &[f64]) -> f64 {
|
||||
let n = x.len() as f64;
|
||||
10.0 * n
|
||||
+ x.iter()
|
||||
.map(|xi| xi * xi - 10.0 * (2.0 * std::f64::consts::PI * xi).cos())
|
||||
.sum::<f64>()
|
||||
}
|
||||
}
|
||||
|
||||
fn run_convergence(
|
||||
name: &str,
|
||||
sampler_name: &str,
|
||||
study: Study<f64>,
|
||||
params: &[FloatParam],
|
||||
objective: fn(&[f64]) -> f64,
|
||||
n_trials: usize,
|
||||
) {
|
||||
let start = Instant::now();
|
||||
|
||||
study
|
||||
.optimize_with_callback(
|
||||
n_trials,
|
||||
|trial| {
|
||||
let x: Vec<f64> = params
|
||||
.iter()
|
||||
.map(|p| p.suggest(trial))
|
||||
.collect::<Result<_, _>>()
|
||||
.unwrap();
|
||||
Ok::<_, optimizer::Error>(objective(&x))
|
||||
},
|
||||
|study, _trial| {
|
||||
let elapsed = start.elapsed().as_millis();
|
||||
let best = study.best_value().unwrap();
|
||||
let n = study.n_trials();
|
||||
println!("{n},{best},{elapsed},{sampler_name},{name}");
|
||||
ControlFlow::Continue(())
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn main() {
|
||||
println!("trial,best_value,elapsed_ms,sampler,function");
|
||||
|
||||
let dims = 5;
|
||||
let params: Vec<FloatParam> = (0..dims)
|
||||
.map(|i| FloatParam::new(-5.0, 5.0).name(format!("x{i}")))
|
||||
.collect();
|
||||
let n_trials = 200;
|
||||
|
||||
// Sphere: Random vs TPE
|
||||
run_convergence(
|
||||
"sphere_5d",
|
||||
"random",
|
||||
Study::minimize(RandomSampler::with_seed(1)),
|
||||
¶ms,
|
||||
functions::sphere,
|
||||
n_trials,
|
||||
);
|
||||
run_convergence(
|
||||
"sphere_5d",
|
||||
"tpe",
|
||||
Study::minimize(TpeSampler::builder().seed(1).build().unwrap()),
|
||||
¶ms,
|
||||
functions::sphere,
|
||||
n_trials,
|
||||
);
|
||||
|
||||
// Rosenbrock: Random vs TPE
|
||||
run_convergence(
|
||||
"rosenbrock_5d",
|
||||
"random",
|
||||
Study::minimize(RandomSampler::with_seed(2)),
|
||||
¶ms,
|
||||
functions::rosenbrock,
|
||||
n_trials,
|
||||
);
|
||||
run_convergence(
|
||||
"rosenbrock_5d",
|
||||
"tpe",
|
||||
Study::minimize(TpeSampler::builder().seed(2).build().unwrap()),
|
||||
¶ms,
|
||||
functions::rosenbrock,
|
||||
n_trials,
|
||||
);
|
||||
|
||||
// Rastrigin: Random vs TPE
|
||||
run_convergence(
|
||||
"rastrigin_5d",
|
||||
"random",
|
||||
Study::minimize(RandomSampler::with_seed(3)),
|
||||
¶ms,
|
||||
functions::rastrigin,
|
||||
n_trials,
|
||||
);
|
||||
run_convergence(
|
||||
"rastrigin_5d",
|
||||
"tpe",
|
||||
Study::minimize(TpeSampler::builder().seed(3).build().unwrap()),
|
||||
¶ms,
|
||||
functions::rastrigin,
|
||||
n_trials,
|
||||
);
|
||||
}
|
||||
@@ -1,275 +0,0 @@
|
||||
//! Machine Learning Hyperparameter Tuning Example
|
||||
//!
|
||||
//! This example shows how to use the optimizer library to find the best
|
||||
//! hyperparameters for a machine learning model. We simulate a gradient
|
||||
//! boosting model (like XGBoost or LightGBM) and search for optimal settings.
|
||||
//!
|
||||
//! # Key Concepts Demonstrated
|
||||
//!
|
||||
//! - Creating a Study with a TPE (Tree-Parzen Estimator) sampler
|
||||
//! - Defining an objective function that the optimizer will minimize
|
||||
//! - Using different parameter types: floats, integers, log-scale, stepped
|
||||
//! - Using callbacks to monitor progress and implement early stopping
|
||||
//!
|
||||
//! # How It Works
|
||||
//!
|
||||
//! 1. Create a `Study` - this manages the optimization process
|
||||
//! 2. Define an objective function that takes a `Trial` and returns a score
|
||||
//! 3. Inside the objective, use `trial.suggest_*()` to sample parameters
|
||||
//! 4. The optimizer runs many trials, learning which parameter regions work best
|
||||
//! 5. After optimization, retrieve the best parameters found
|
||||
//!
|
||||
//! Run with: `cargo run --example ml_hyperparameter_tuning`
|
||||
|
||||
use std::ops::ControlFlow;
|
||||
|
||||
use optimizer::prelude::*;
|
||||
|
||||
// ============================================================================
|
||||
// Configuration: Hyperparameters we want to tune
|
||||
// ============================================================================
|
||||
|
||||
/// Holds all the hyperparameters for our model.
|
||||
///
|
||||
/// In a real application, you would pass these to your ML framework
|
||||
/// (e.g., XGBoost, LightGBM, scikit-learn).
|
||||
struct ModelConfig {
|
||||
learning_rate: f64,
|
||||
max_depth: i64,
|
||||
n_estimators: i64,
|
||||
subsample: f64,
|
||||
colsample_bytree: f64,
|
||||
min_child_weight: i64,
|
||||
reg_alpha: f64,
|
||||
reg_lambda: f64,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Objective Function: What we want to optimize
|
||||
// ============================================================================
|
||||
|
||||
/// Simulates training a model and returns the validation loss.
|
||||
///
|
||||
/// In a real scenario, this function would:
|
||||
/// 1. Create a model with the given hyperparameters
|
||||
/// 2. Train it on your training data
|
||||
/// 3. Evaluate it on validation data
|
||||
/// 4. Return the validation metric (e.g., RMSE, log loss, accuracy)
|
||||
///
|
||||
/// The optimizer will try to MINIMIZE this value (we set Direction::Minimize).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn evaluate_model(config: &ModelConfig) -> f64 {
|
||||
// Simulated optimal hyperparameters:
|
||||
// learning_rate ~ 0.05, max_depth ~ 6, n_estimators ~ 200
|
||||
// subsample ~ 0.8, colsample_bytree ~ 0.8, min_child_weight ~ 3
|
||||
// reg_alpha ~ 0.1, reg_lambda ~ 1.0
|
||||
|
||||
let mut loss = 0.15; // Base loss
|
||||
|
||||
// Each term penalizes deviation from the optimal value
|
||||
loss += (config.learning_rate - 0.05).powi(2) * 100.0;
|
||||
loss += ((config.max_depth - 6) as f64).powi(2) * 0.01;
|
||||
loss += ((config.n_estimators - 200) as f64).powi(2) * 0.00001;
|
||||
loss += (config.subsample - 0.8).powi(2) * 10.0;
|
||||
loss += (config.colsample_bytree - 0.8).powi(2) * 10.0;
|
||||
loss += ((config.min_child_weight - 3) as f64).powi(2) * 0.05;
|
||||
loss += (config.reg_alpha - 0.1).powi(2) * 5.0;
|
||||
loss += (config.reg_lambda - 1.0).powi(2) * 2.0;
|
||||
|
||||
// Add some noise to simulate real-world variability
|
||||
let noise = (config.learning_rate * 1000.0).sin() * 0.01;
|
||||
|
||||
loss + noise
|
||||
}
|
||||
|
||||
/// The objective function that the optimizer calls for each trial.
|
||||
///
|
||||
/// This function:
|
||||
/// 1. Uses parameter definitions passed as arguments
|
||||
/// 2. Builds a model configuration from the suggested values
|
||||
/// 3. Evaluates the model and returns the loss
|
||||
///
|
||||
/// The optimizer learns from the results to suggest better parameters
|
||||
/// in future trials.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn objective(
|
||||
trial: &mut Trial,
|
||||
learning_rate_param: &FloatParam,
|
||||
max_depth_param: &IntParam,
|
||||
n_estimators_param: &IntParam,
|
||||
subsample_param: &FloatParam,
|
||||
colsample_bytree_param: &FloatParam,
|
||||
min_child_weight_param: &IntParam,
|
||||
reg_alpha_param: &FloatParam,
|
||||
reg_lambda_param: &FloatParam,
|
||||
) -> optimizer::Result<f64> {
|
||||
let learning_rate = learning_rate_param.suggest(trial)?;
|
||||
let max_depth = max_depth_param.suggest(trial)?;
|
||||
let n_estimators = n_estimators_param.suggest(trial)?;
|
||||
let subsample = subsample_param.suggest(trial)?;
|
||||
let colsample_bytree = colsample_bytree_param.suggest(trial)?;
|
||||
let min_child_weight = min_child_weight_param.suggest(trial)?;
|
||||
let reg_alpha = reg_alpha_param.suggest(trial)?;
|
||||
let reg_lambda = reg_lambda_param.suggest(trial)?;
|
||||
|
||||
// Build configuration and evaluate
|
||||
let config = ModelConfig {
|
||||
learning_rate,
|
||||
max_depth,
|
||||
n_estimators,
|
||||
subsample,
|
||||
colsample_bytree,
|
||||
min_child_weight,
|
||||
reg_alpha,
|
||||
reg_lambda,
|
||||
};
|
||||
|
||||
let loss = evaluate_model(&config);
|
||||
|
||||
Ok(loss)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Callback Function: Monitor progress and implement early stopping
|
||||
// ============================================================================
|
||||
|
||||
/// Called after each successful trial completes.
|
||||
///
|
||||
/// Use callbacks to:
|
||||
/// - Log progress to console or file
|
||||
/// - Save checkpoints
|
||||
/// - Implement early stopping when a good solution is found
|
||||
/// - Track metrics over time
|
||||
///
|
||||
/// Return `ControlFlow::Continue(())` to keep optimizing.
|
||||
/// Return `ControlFlow::Break(())` to stop early.
|
||||
fn on_trial_complete(study: &Study<f64>, trial: &CompletedTrial<f64>) -> ControlFlow<()> {
|
||||
// Print trial number and objective value
|
||||
print!("{:>5} ", study.n_trials());
|
||||
for value in trial.params.values() {
|
||||
print!("{value:>12} ");
|
||||
}
|
||||
println!("{:>12.6}", trial.value);
|
||||
|
||||
// Early stopping: if we find an excellent solution, stop early
|
||||
if trial.value < 0.16 {
|
||||
println!("\nEarly stopping: found excellent solution!");
|
||||
return ControlFlow::Break(());
|
||||
}
|
||||
|
||||
ControlFlow::Continue(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Main: Set up and run the optimization
|
||||
// ============================================================================
|
||||
|
||||
fn main() -> optimizer::Result<()> {
|
||||
println!("=== ML Hyperparameter Tuning Example ===\n");
|
||||
|
||||
// Step 1: Create a sampler
|
||||
//
|
||||
// TPE (Tree-Parzen Estimator) is a Bayesian optimization algorithm.
|
||||
// It learns from previous trials to suggest better parameters.
|
||||
// - n_startup_trials: Number of random trials before TPE kicks in
|
||||
// - gamma: What fraction of trials are considered "good" (lower = more selective)
|
||||
// - seed: For reproducibility
|
||||
let sampler = TpeSampler::builder()
|
||||
.n_startup_trials(10)
|
||||
.gamma(0.25)
|
||||
.seed(42)
|
||||
.build()
|
||||
.expect("Failed to build TPE sampler");
|
||||
|
||||
// Step 2: Create a study
|
||||
//
|
||||
// The study manages the optimization process. We want to MINIMIZE
|
||||
// the loss (lower is better). Use Direction::Maximize for metrics
|
||||
// where higher is better (like accuracy).
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
// Print header
|
||||
println!("Starting hyperparameter optimization...\n");
|
||||
println!(
|
||||
"{:>5} {:>12} (parameters...) {:>12}",
|
||||
"Trial", "Params", "Loss"
|
||||
);
|
||||
println!("{}", "-".repeat(60));
|
||||
|
||||
// Step 3: Define parameter search spaces
|
||||
let learning_rate_param = FloatParam::new(0.001, 0.3)
|
||||
.name("learning_rate")
|
||||
.log_scale();
|
||||
let max_depth_param = IntParam::new(3, 12).name("max_depth");
|
||||
let n_estimators_param = IntParam::new(50, 500).name("n_estimators").step(50);
|
||||
let subsample_param = FloatParam::new(0.5, 1.0).name("subsample");
|
||||
let colsample_bytree_param = FloatParam::new(0.5, 1.0).name("colsample_bytree");
|
||||
let min_child_weight_param = IntParam::new(1, 10).name("min_child_weight");
|
||||
let reg_alpha_param = FloatParam::new(1e-3, 10.0).name("reg_alpha").log_scale();
|
||||
let reg_lambda_param = FloatParam::new(1e-3, 10.0).name("reg_lambda").log_scale();
|
||||
|
||||
// Step 4: Run optimization
|
||||
//
|
||||
// optimize_with_callback runs the objective function for up to
|
||||
// n_trials iterations. After each trial, it calls the callback.
|
||||
// The sampler gets access to trial history for informed sampling.
|
||||
let n_trials = 50;
|
||||
|
||||
study.optimize_with_callback(
|
||||
n_trials,
|
||||
|trial| {
|
||||
objective(
|
||||
trial,
|
||||
&learning_rate_param,
|
||||
&max_depth_param,
|
||||
&n_estimators_param,
|
||||
&subsample_param,
|
||||
&colsample_bytree_param,
|
||||
&min_child_weight_param,
|
||||
®_alpha_param,
|
||||
®_lambda_param,
|
||||
)
|
||||
},
|
||||
on_trial_complete,
|
||||
)?;
|
||||
|
||||
// Step 4: Get the best result
|
||||
println!("\n{}", "=".repeat(110));
|
||||
println!("\nOptimization completed!");
|
||||
println!("Total trials: {}", study.n_trials());
|
||||
|
||||
let best = study.best_trial()?;
|
||||
println!("\nBest trial:");
|
||||
println!(" Loss: {:.6}", best.value);
|
||||
println!(" Parameters:");
|
||||
println!(
|
||||
" learning_rate: {:.6}",
|
||||
best.get(&learning_rate_param).unwrap()
|
||||
);
|
||||
println!(" max_depth: {}", best.get(&max_depth_param).unwrap());
|
||||
println!(
|
||||
" n_estimators: {}",
|
||||
best.get(&n_estimators_param).unwrap()
|
||||
);
|
||||
println!(" subsample: {:.6}", best.get(&subsample_param).unwrap());
|
||||
println!(
|
||||
" colsample_bytree: {:.6}",
|
||||
best.get(&colsample_bytree_param).unwrap()
|
||||
);
|
||||
println!(
|
||||
" min_child_weight: {}",
|
||||
best.get(&min_child_weight_param).unwrap()
|
||||
);
|
||||
println!(" reg_alpha: {:.6}", best.get(®_alpha_param).unwrap());
|
||||
println!(
|
||||
" reg_lambda: {:.6}",
|
||||
best.get(®_lambda_param).unwrap()
|
||||
);
|
||||
|
||||
// Step 5: Use the best parameters (in a real app)
|
||||
//
|
||||
// Now you would take best.params and use them to train your final model
|
||||
// on the full dataset.
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
use optimizer::prelude::*;
|
||||
use optimizer_derive::Categorical;
|
||||
|
||||
#[derive(Clone, Debug, Categorical)]
|
||||
enum Activation {
|
||||
Relu,
|
||||
Sigmoid,
|
||||
Tanh,
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
|
||||
// Define parameters outside the objective function
|
||||
let lr_param = FloatParam::new(1e-5, 1e-1).name("lr").log_scale();
|
||||
let n_layers_param = IntParam::new(1, 5).name("n_layers");
|
||||
let units_param = IntParam::new(32, 512).name("units").step(32);
|
||||
let optimizer_param = CategoricalParam::new(vec!["sgd", "adam", "rmsprop"]).name("optimizer");
|
||||
let activation_param = EnumParam::<Activation>::new().name("activation");
|
||||
let batch_size_param = IntParam::new(16, 256).name("batch_size").log_scale();
|
||||
let use_dropout_param = BoolParam::new().name("use_dropout");
|
||||
|
||||
study
|
||||
.optimize(20, |trial| {
|
||||
let lr = lr_param.suggest(trial)?;
|
||||
let n_layers = n_layers_param.suggest(trial)?;
|
||||
let units = units_param.suggest(trial)?;
|
||||
let optimizer = optimizer_param.suggest(trial)?;
|
||||
let use_dropout = use_dropout_param.suggest(trial)?;
|
||||
let activation = activation_param.suggest(trial)?;
|
||||
let batch_size = batch_size_param.suggest(trial)?;
|
||||
|
||||
// Simulate a loss function
|
||||
let loss = lr * (n_layers as f64) + (units as f64) * 0.001
|
||||
- if use_dropout { 0.1 } else { 0.0 };
|
||||
|
||||
println!(
|
||||
"Trial {}: lr={lr:.6}, layers={n_layers}, units={units}, opt={optimizer}, \
|
||||
dropout={use_dropout}, activation={activation:?}, batch={batch_size} -> loss={loss:.4}",
|
||||
trial.id()
|
||||
);
|
||||
|
||||
Ok::<_, Error>(loss)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let best = study.best_trial().unwrap();
|
||||
println!("\nBest trial: value={:.4}", best.value);
|
||||
println!(" lr: {:.6}", best.get(&lr_param).unwrap());
|
||||
println!(" n_layers: {}", best.get(&n_layers_param).unwrap());
|
||||
println!(" units: {}", best.get(&units_param).unwrap());
|
||||
println!(" optimizer: {}", best.get(&optimizer_param).unwrap());
|
||||
println!(" activation: {:?}", best.get(&activation_param).unwrap());
|
||||
println!(" batch_size: {}", best.get(&batch_size_param).unwrap());
|
||||
println!(" use_dropout: {}", best.get(&use_dropout_param).unwrap());
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
use optimizer::prelude::*;
|
||||
|
||||
fn main() {
|
||||
// Multi-parameter optimization with TPE sampler.
|
||||
let sampler = TpeSampler::builder().seed(42).build().unwrap();
|
||||
let mut study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
study.set_pruner(MedianPruner::new(Direction::Minimize));
|
||||
|
||||
let lr = FloatParam::new(1e-5, 1e-1)
|
||||
.log_scale()
|
||||
.name("learning_rate");
|
||||
let n_layers = IntParam::new(1, 5).name("n_layers");
|
||||
let dropout = FloatParam::new(0.0, 0.5).step(0.05).name("dropout");
|
||||
let batch_size = CategoricalParam::new(vec![16, 32, 64, 128]).name("batch_size");
|
||||
|
||||
study
|
||||
.optimize(80, |trial| {
|
||||
let lr_val = lr.suggest(trial)?;
|
||||
let layers = n_layers.suggest(trial)?;
|
||||
let drop = dropout.suggest(trial)?;
|
||||
let bs = batch_size.suggest(trial)?;
|
||||
|
||||
// Simulate training with intermediate reporting.
|
||||
let mut loss = 1.0;
|
||||
for epoch in 0..10 {
|
||||
loss *= 0.7 + 0.3 * lr_val.ln().abs() / 12.0;
|
||||
loss += drop * 0.05;
|
||||
loss += (1.0 / bs as f64) * 0.1;
|
||||
loss -= layers as f64 * 0.02;
|
||||
trial.report(epoch, loss);
|
||||
if trial.should_prune() {
|
||||
return Err(TrialPruned.into());
|
||||
}
|
||||
}
|
||||
|
||||
Ok::<_, Error>(loss)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
println!("{}", study.summary());
|
||||
|
||||
let path = "optimization_report.html";
|
||||
generate_html_report(&study, path).unwrap();
|
||||
println!("\nReport saved to {path}");
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
[package]
|
||||
name = "optimizer-derive"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.88"
|
||||
license = "MIT"
|
||||
description = "Derive macros for the optimizer crate"
|
||||
repository = "https://github.com/raimannma/rust-optimizer"
|
||||
|
||||
[lib]
|
||||
proc-macro = true
|
||||
|
||||
[dependencies]
|
||||
syn = { version = "2", features = ["full"] }
|
||||
quote = "1"
|
||||
@@ -1,71 +0,0 @@
|
||||
use proc_macro::TokenStream;
|
||||
use quote::quote;
|
||||
use syn::{Data, DeriveInput, Fields, parse_macro_input};
|
||||
|
||||
/// Derive macro for the `Categorical` trait on fieldless enums.
|
||||
///
|
||||
/// Generates an implementation of `optimizer::Categorical` that maps
|
||||
/// enum variants to/from sequential indices.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```ignore
|
||||
/// use optimizer::Categorical;
|
||||
///
|
||||
/// #[derive(Clone, Categorical)]
|
||||
/// enum Color {
|
||||
/// Red,
|
||||
/// Green,
|
||||
/// Blue,
|
||||
/// }
|
||||
/// ```
|
||||
#[proc_macro_derive(Categorical)]
|
||||
pub fn derive_categorical(input: TokenStream) -> TokenStream {
|
||||
let input = parse_macro_input!(input as DeriveInput);
|
||||
let name = &input.ident;
|
||||
|
||||
let Data::Enum(data_enum) = &input.data else {
|
||||
return syn::Error::new_spanned(&input, "Categorical can only be derived for enums")
|
||||
.to_compile_error()
|
||||
.into();
|
||||
};
|
||||
|
||||
// Validate all variants are fieldless
|
||||
for variant in &data_enum.variants {
|
||||
if !matches!(variant.fields, Fields::Unit) {
|
||||
return syn::Error::new_spanned(
|
||||
variant,
|
||||
"Categorical can only be derived for enums with unit variants (no fields)",
|
||||
)
|
||||
.to_compile_error()
|
||||
.into();
|
||||
}
|
||||
}
|
||||
|
||||
let n_choices = data_enum.variants.len();
|
||||
let variant_names: Vec<_> = data_enum.variants.iter().map(|v| &v.ident).collect();
|
||||
let indices: Vec<usize> = (0..n_choices).collect();
|
||||
|
||||
let (impl_generics, ty_generics, where_clause) = input.generics.split_for_impl();
|
||||
|
||||
let expanded = quote! {
|
||||
impl #impl_generics optimizer::Categorical for #name #ty_generics #where_clause {
|
||||
const N_CHOICES: usize = #n_choices;
|
||||
|
||||
fn from_index(index: usize) -> Self {
|
||||
match index {
|
||||
#(#indices => #name::#variant_names,)*
|
||||
_ => panic!("invalid index {} for {} with {} variants", index, stringify!(#name), #n_choices),
|
||||
}
|
||||
}
|
||||
|
||||
fn to_index(&self) -> usize {
|
||||
match self {
|
||||
#(#name::#variant_names => #indices,)*
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
expanded.into()
|
||||
}
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
/// Distribution for floating-point parameters.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
pub struct FloatDistribution {
|
||||
/// Lower bound (inclusive).
|
||||
pub low: f64,
|
||||
@@ -16,7 +15,6 @@ pub struct FloatDistribution {
|
||||
|
||||
/// Distribution for integer parameters.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
pub struct IntDistribution {
|
||||
/// Lower bound (inclusive).
|
||||
pub low: i64,
|
||||
@@ -30,7 +28,6 @@ pub struct IntDistribution {
|
||||
|
||||
/// Distribution for categorical parameters.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
pub struct CategoricalDistribution {
|
||||
/// Number of choices available.
|
||||
pub n_choices: usize,
|
||||
@@ -38,7 +35,6 @@ pub struct CategoricalDistribution {
|
||||
|
||||
/// Enum wrapping all parameter distribution types.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
pub enum Distribution {
|
||||
/// A floating-point distribution.
|
||||
Float(FloatDistribution),
|
||||
|
||||
+9
-93
@@ -1,5 +1,10 @@
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
//! Error types for the optimizer library.
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// The error type for TPE operations.
|
||||
#[derive(Debug, Error)]
|
||||
pub enum TpeError {
|
||||
/// Returned when the lower bound is greater than the upper bound.
|
||||
#[error("invalid bounds: low ({low}) must be less than or equal to high ({high})")]
|
||||
InvalidBounds {
|
||||
@@ -33,96 +38,7 @@ pub enum Error {
|
||||
/// Returned when requesting the best trial but no trials have completed.
|
||||
#[error("no completed trials available")]
|
||||
NoCompletedTrials,
|
||||
|
||||
/// Returned when gamma is not in the valid range (0.0, 1.0).
|
||||
#[error("invalid gamma: {0} must be in (0.0, 1.0)")]
|
||||
InvalidGamma(f64),
|
||||
|
||||
/// Returned when bandwidth is not positive.
|
||||
#[error("invalid bandwidth: {0} must be positive")]
|
||||
InvalidBandwidth(f64),
|
||||
|
||||
/// Returned when KDE is created with empty samples.
|
||||
#[error("KDE requires at least one sample")]
|
||||
EmptySamples,
|
||||
|
||||
/// Returned when multivariate KDE samples have zero dimensions.
|
||||
#[error("multivariate KDE samples must have at least one dimension")]
|
||||
ZeroDimensions,
|
||||
|
||||
/// Returned when multivariate KDE samples have inconsistent dimensions.
|
||||
#[error(
|
||||
"dimension mismatch: expected {expected} dimensions but sample {sample_index} has {got}"
|
||||
)]
|
||||
DimensionMismatch {
|
||||
/// The expected number of dimensions.
|
||||
expected: usize,
|
||||
/// The actual number of dimensions in the sample.
|
||||
got: usize,
|
||||
/// The index of the sample with mismatched dimensions.
|
||||
sample_index: usize,
|
||||
},
|
||||
|
||||
/// Returned when bandwidth vector length doesn't match the number of dimensions.
|
||||
#[error("bandwidth dimension mismatch: expected {expected} bandwidths but got {got}")]
|
||||
BandwidthDimensionMismatch {
|
||||
/// The expected number of bandwidths.
|
||||
expected: usize,
|
||||
/// The actual number of bandwidths provided.
|
||||
got: usize,
|
||||
},
|
||||
|
||||
/// Returned when a trial is pruned (stopped early by the objective function).
|
||||
#[error("trial was pruned")]
|
||||
TrialPruned,
|
||||
|
||||
/// Returned when the objective returns the wrong number of values.
|
||||
#[error("objective dimension mismatch: expected {expected} values, got {got}")]
|
||||
ObjectiveDimensionMismatch {
|
||||
/// The expected number of objective values.
|
||||
expected: usize,
|
||||
/// The actual number of objective values returned.
|
||||
got: usize,
|
||||
},
|
||||
|
||||
/// Returned when an internal invariant is violated.
|
||||
#[error("internal error: {0}")]
|
||||
Internal(&'static str),
|
||||
|
||||
/// Returned when an async task fails.
|
||||
#[cfg(feature = "async")]
|
||||
#[error("async task error: {0}")]
|
||||
TaskError(String),
|
||||
}
|
||||
|
||||
pub type Result<T> = core::result::Result<T, Error>;
|
||||
|
||||
/// Convenience type for signalling a pruned trial from an objective function.
|
||||
///
|
||||
/// Implements `Into<Error>` so it can be used with `?` in objectives that
|
||||
/// return `Result<V, Error>`.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::{Error, TrialPruned};
|
||||
///
|
||||
/// fn objective_that_prunes() -> Result<f64, Error> {
|
||||
/// // ... some computation ...
|
||||
/// Err(TrialPruned)?
|
||||
/// }
|
||||
/// ```
|
||||
#[derive(Debug)]
|
||||
pub struct TrialPruned;
|
||||
|
||||
impl core::fmt::Display for TrialPruned {
|
||||
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
|
||||
write!(f, "trial was pruned")
|
||||
}
|
||||
}
|
||||
|
||||
impl From<TrialPruned> for Error {
|
||||
fn from(_: TrialPruned) -> Self {
|
||||
Error::TrialPruned
|
||||
}
|
||||
}
|
||||
/// A specialized Result type for TPE operations.
|
||||
pub type Result<T> = std::result::Result<T, TpeError>;
|
||||
|
||||
-536
@@ -1,536 +0,0 @@
|
||||
//! fANOVA (functional ANOVA) parameter importance via random forest.
|
||||
//!
|
||||
//! Decomposes the variance of the objective function into contributions
|
||||
//! from individual parameters (main effects) and parameter interactions.
|
||||
//!
|
||||
//! The algorithm:
|
||||
//! 1. Fits a random forest to `(parameters) -> objective_value`
|
||||
//! 2. Applies functional ANOVA decomposition to the forest
|
||||
//! 3. Computes main effects (single-parameter importance)
|
||||
//! 4. Computes interaction effects (pairwise parameter importance)
|
||||
|
||||
use rand::rngs::StdRng;
|
||||
use rand::{RngExt, SeedableRng};
|
||||
|
||||
/// Result of fANOVA analysis.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FanovaResult {
|
||||
/// Per-parameter importance (fraction of total variance explained).
|
||||
/// Sorted by descending importance.
|
||||
pub main_effects: Vec<(String, f64)>,
|
||||
/// Pairwise interaction importance (fraction of total variance explained).
|
||||
/// Sorted by descending importance.
|
||||
pub interactions: Vec<((String, String), f64)>,
|
||||
}
|
||||
|
||||
/// Configuration for fANOVA analysis.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FanovaConfig {
|
||||
/// Number of trees in the random forest (default: 64).
|
||||
pub n_trees: usize,
|
||||
/// Maximum depth of each tree. `None` for unlimited (default: `None`).
|
||||
pub max_depth: Option<usize>,
|
||||
/// Minimum samples required to split a node (default: 2).
|
||||
pub min_samples_split: usize,
|
||||
/// Minimum samples required in a leaf node (default: 1).
|
||||
pub min_samples_leaf: usize,
|
||||
/// Random seed for reproducibility (default: `Some(42)`).
|
||||
pub seed: Option<u64>,
|
||||
}
|
||||
|
||||
impl Default for FanovaConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
n_trees: 64,
|
||||
max_depth: None,
|
||||
min_samples_split: 2,
|
||||
min_samples_leaf: 1,
|
||||
seed: Some(42),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Decision Tree ---
|
||||
|
||||
/// A node in the regression tree (arena-allocated).
|
||||
#[derive(Debug, Clone)]
|
||||
enum TreeNode {
|
||||
Leaf {
|
||||
value: f64,
|
||||
n_samples: usize,
|
||||
},
|
||||
Split {
|
||||
feature: usize,
|
||||
threshold: f64,
|
||||
left: usize,
|
||||
right: usize,
|
||||
n_samples: usize,
|
||||
},
|
||||
}
|
||||
|
||||
/// A regression decision tree for fANOVA.
|
||||
#[derive(Debug, Clone)]
|
||||
struct DecisionTree {
|
||||
nodes: Vec<TreeNode>,
|
||||
}
|
||||
|
||||
impl DecisionTree {
|
||||
/// Build a tree from the given data using the specified bootstrap indices.
|
||||
fn build(
|
||||
data: &[Vec<f64>],
|
||||
targets: &[f64],
|
||||
indices: &[usize],
|
||||
config: &FanovaConfig,
|
||||
rng: &mut StdRng,
|
||||
) -> Self {
|
||||
let mut tree = Self { nodes: Vec::new() };
|
||||
tree.build_node(data, targets, indices, 0, config, rng);
|
||||
tree
|
||||
}
|
||||
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
fn build_node(
|
||||
&mut self,
|
||||
data: &[Vec<f64>],
|
||||
targets: &[f64],
|
||||
indices: &[usize],
|
||||
depth: usize,
|
||||
config: &FanovaConfig,
|
||||
rng: &mut StdRng,
|
||||
) -> usize {
|
||||
let n = indices.len();
|
||||
let mean = indices.iter().map(|&i| targets[i]).sum::<f64>() / n as f64;
|
||||
|
||||
// Stopping conditions
|
||||
if n < config.min_samples_split || config.max_depth.is_some_and(|d| depth >= d) {
|
||||
let idx = self.nodes.len();
|
||||
self.nodes.push(TreeNode::Leaf {
|
||||
value: mean,
|
||||
n_samples: n,
|
||||
});
|
||||
return idx;
|
||||
}
|
||||
|
||||
// Pure node check (all targets identical)
|
||||
#[allow(clippy::float_cmp)]
|
||||
if indices.iter().all(|&i| targets[i] == targets[indices[0]]) {
|
||||
let idx = self.nodes.len();
|
||||
self.nodes.push(TreeNode::Leaf {
|
||||
value: mean,
|
||||
n_samples: n,
|
||||
});
|
||||
return idx;
|
||||
}
|
||||
|
||||
let n_features = data[0].len();
|
||||
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
|
||||
let max_features = ((n_features as f64).sqrt().ceil() as usize)
|
||||
.max(1)
|
||||
.min(n_features);
|
||||
let candidates = partial_shuffle(n_features, max_features, rng);
|
||||
|
||||
// Total variance at this node
|
||||
let total_var: f64 = indices.iter().map(|&i| (targets[i] - mean).powi(2)).sum();
|
||||
if total_var == 0.0 {
|
||||
let idx = self.nodes.len();
|
||||
self.nodes.push(TreeNode::Leaf {
|
||||
value: mean,
|
||||
n_samples: n,
|
||||
});
|
||||
return idx;
|
||||
}
|
||||
|
||||
let mut best_score = f64::NEG_INFINITY;
|
||||
let mut best_feature = 0;
|
||||
let mut best_threshold = 0.0;
|
||||
|
||||
for &feat in &candidates {
|
||||
let mut values: Vec<f64> = indices.iter().map(|&i| data[i][feat]).collect();
|
||||
values.sort_by(|a, b| a.partial_cmp(b).unwrap_or(core::cmp::Ordering::Equal));
|
||||
values.dedup();
|
||||
|
||||
if values.len() < 2 {
|
||||
continue;
|
||||
}
|
||||
|
||||
for w in values.windows(2) {
|
||||
let threshold = f64::midpoint(w[0], w[1]);
|
||||
let (l_sum, l_sq, l_n, r_sum, r_sq, r_n) =
|
||||
split_stats(data, targets, indices, feat, threshold);
|
||||
|
||||
if l_n < config.min_samples_leaf || r_n < config.min_samples_leaf {
|
||||
continue;
|
||||
}
|
||||
|
||||
let l_var = l_sq - l_sum * l_sum / l_n as f64;
|
||||
let r_var = r_sq - r_sum * r_sum / r_n as f64;
|
||||
let score = total_var - l_var - r_var;
|
||||
|
||||
if score > best_score {
|
||||
best_score = score;
|
||||
best_feature = feat;
|
||||
best_threshold = threshold;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if best_score <= 0.0 {
|
||||
let idx = self.nodes.len();
|
||||
self.nodes.push(TreeNode::Leaf {
|
||||
value: mean,
|
||||
n_samples: n,
|
||||
});
|
||||
return idx;
|
||||
}
|
||||
|
||||
let (left_indices, right_indices): (Vec<usize>, Vec<usize>) = indices
|
||||
.iter()
|
||||
.partition(|&&i| data[i][best_feature] <= best_threshold);
|
||||
|
||||
if left_indices.is_empty() || right_indices.is_empty() {
|
||||
let idx = self.nodes.len();
|
||||
self.nodes.push(TreeNode::Leaf {
|
||||
value: mean,
|
||||
n_samples: n,
|
||||
});
|
||||
return idx;
|
||||
}
|
||||
|
||||
// Reserve slot for this split node (placeholder replaced below)
|
||||
let node_idx = self.nodes.len();
|
||||
self.nodes.push(TreeNode::Leaf {
|
||||
value: 0.0,
|
||||
n_samples: 0,
|
||||
});
|
||||
|
||||
let left = self.build_node(data, targets, &left_indices, depth + 1, config, rng);
|
||||
let right = self.build_node(data, targets, &right_indices, depth + 1, config, rng);
|
||||
|
||||
self.nodes[node_idx] = TreeNode::Split {
|
||||
feature: best_feature,
|
||||
threshold: best_threshold,
|
||||
left,
|
||||
right,
|
||||
n_samples: n,
|
||||
};
|
||||
|
||||
node_idx
|
||||
}
|
||||
|
||||
/// Compute marginal prediction for a given feature subset.
|
||||
///
|
||||
/// Features in `subset` use values from `feature_values`.
|
||||
/// Features not in `subset` are marginalized by weighting branches
|
||||
/// proportionally to their training-data fractions.
|
||||
fn marginal_predict(&self, subset: &[usize], feature_values: &[f64]) -> f64 {
|
||||
self.marginal_predict_at(0, subset, feature_values)
|
||||
}
|
||||
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
fn marginal_predict_at(&self, idx: usize, subset: &[usize], vals: &[f64]) -> f64 {
|
||||
match self.nodes[idx] {
|
||||
TreeNode::Leaf { value, .. } => value,
|
||||
TreeNode::Split {
|
||||
feature,
|
||||
threshold,
|
||||
left,
|
||||
right,
|
||||
n_samples,
|
||||
} => {
|
||||
if subset.contains(&feature) {
|
||||
if vals[feature] <= threshold {
|
||||
self.marginal_predict_at(left, subset, vals)
|
||||
} else {
|
||||
self.marginal_predict_at(right, subset, vals)
|
||||
}
|
||||
} else {
|
||||
let l_n = self.n_samples(left) as f64;
|
||||
let r_n = self.n_samples(right) as f64;
|
||||
let total = n_samples as f64;
|
||||
(l_n / total) * self.marginal_predict_at(left, subset, vals)
|
||||
+ (r_n / total) * self.marginal_predict_at(right, subset, vals)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn n_samples(&self, idx: usize) -> usize {
|
||||
match self.nodes[idx] {
|
||||
TreeNode::Leaf { n_samples, .. } | TreeNode::Split { n_samples, .. } => n_samples,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Helper Functions ---
|
||||
|
||||
/// Select `k` random indices from `0..n` using partial Fisher-Yates shuffle.
|
||||
fn partial_shuffle(n: usize, k: usize, rng: &mut StdRng) -> Vec<usize> {
|
||||
let mut indices: Vec<usize> = (0..n).collect();
|
||||
let k = k.min(n);
|
||||
for i in 0..k {
|
||||
let j = rng.random_range(i..n);
|
||||
indices.swap(i, j);
|
||||
}
|
||||
indices.truncate(k);
|
||||
indices
|
||||
}
|
||||
|
||||
/// Compute left/right split statistics for variance reduction.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
fn split_stats(
|
||||
data: &[Vec<f64>],
|
||||
targets: &[f64],
|
||||
indices: &[usize],
|
||||
feature: usize,
|
||||
threshold: f64,
|
||||
) -> (f64, f64, usize, f64, f64, usize) {
|
||||
let (mut l_sum, mut l_sq, mut l_n) = (0.0, 0.0, 0usize);
|
||||
let (mut r_sum, mut r_sq, mut r_n) = (0.0, 0.0, 0usize);
|
||||
|
||||
for &i in indices {
|
||||
let y = targets[i];
|
||||
if data[i][feature] <= threshold {
|
||||
l_sum += y;
|
||||
l_sq += y * y;
|
||||
l_n += 1;
|
||||
} else {
|
||||
r_sum += y;
|
||||
r_sq += y * y;
|
||||
r_n += 1;
|
||||
}
|
||||
}
|
||||
|
||||
(l_sum, l_sq, l_n, r_sum, r_sq, r_n)
|
||||
}
|
||||
|
||||
/// Population variance of a slice.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
fn variance(values: &[f64]) -> f64 {
|
||||
if values.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
let n = values.len() as f64;
|
||||
let mean = values.iter().sum::<f64>() / n;
|
||||
values.iter().map(|v| (v - mean).powi(2)).sum::<f64>() / n
|
||||
}
|
||||
|
||||
// --- Public API ---
|
||||
|
||||
/// Run fANOVA analysis on pre-processed numerical data.
|
||||
///
|
||||
/// `data` is `n_samples` rows, each with `n_features` columns.
|
||||
/// `targets` has one entry per sample.
|
||||
/// `feature_names` maps feature index to human-readable name.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
pub(crate) fn compute_fanova(
|
||||
data: &[Vec<f64>],
|
||||
targets: &[f64],
|
||||
feature_names: &[String],
|
||||
config: &FanovaConfig,
|
||||
) -> FanovaResult {
|
||||
let n_samples = data.len();
|
||||
let n_features = data[0].len();
|
||||
|
||||
let mut rng: StdRng = config
|
||||
.seed
|
||||
.map_or_else(rand::make_rng, StdRng::seed_from_u64);
|
||||
|
||||
// Build random forest with bootstrap sampling
|
||||
let trees: Vec<DecisionTree> = (0..config.n_trees)
|
||||
.map(|_| {
|
||||
let bootstrap: Vec<usize> = (0..n_samples)
|
||||
.map(|_| rng.random_range(0..n_samples))
|
||||
.collect();
|
||||
DecisionTree::build(data, targets, &bootstrap, config, &mut rng)
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Compute main effects: V_j = Var[E[f | x_j]]
|
||||
let main_var: Vec<f64> = (0..n_features)
|
||||
.map(|j| {
|
||||
let subset = [j];
|
||||
let preds: Vec<f64> = (0..n_samples)
|
||||
.map(|i| {
|
||||
trees
|
||||
.iter()
|
||||
.map(|t| t.marginal_predict(&subset, &data[i]))
|
||||
.sum::<f64>()
|
||||
/ trees.len() as f64
|
||||
})
|
||||
.collect();
|
||||
variance(&preds)
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Compute pairwise interaction effects: V_{j,k} - V_j - V_k
|
||||
let mut interactions: Vec<((String, String), f64)> = Vec::new();
|
||||
for j in 0..n_features {
|
||||
for k in (j + 1)..n_features {
|
||||
let subset = [j, k];
|
||||
let preds: Vec<f64> = (0..n_samples)
|
||||
.map(|i| {
|
||||
trees
|
||||
.iter()
|
||||
.map(|t| t.marginal_predict(&subset, &data[i]))
|
||||
.sum::<f64>()
|
||||
/ trees.len() as f64
|
||||
})
|
||||
.collect();
|
||||
let joint = variance(&preds);
|
||||
let interaction = (joint - main_var[j] - main_var[k]).max(0.0);
|
||||
if interaction > 1e-10 {
|
||||
interactions.push((
|
||||
(feature_names[j].clone(), feature_names[k].clone()),
|
||||
interaction,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize so all importances sum to 1.0
|
||||
let total: f64 =
|
||||
main_var.iter().sum::<f64>() + interactions.iter().map(|(_, v)| *v).sum::<f64>();
|
||||
|
||||
let mut main_effects: Vec<(String, f64)> = feature_names
|
||||
.iter()
|
||||
.zip(&main_var)
|
||||
.map(|(name, &v)| (name.clone(), if total > 0.0 { v / total } else { 0.0 }))
|
||||
.collect();
|
||||
main_effects.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(core::cmp::Ordering::Equal));
|
||||
|
||||
if total > 0.0 {
|
||||
for entry in &mut interactions {
|
||||
entry.1 /= total;
|
||||
}
|
||||
}
|
||||
interactions.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(core::cmp::Ordering::Equal));
|
||||
|
||||
FanovaResult {
|
||||
main_effects,
|
||||
interactions,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn single_dominant_parameter() {
|
||||
// f(x, y) = x — only x matters
|
||||
let mut rng = StdRng::seed_from_u64(0);
|
||||
let n = 100;
|
||||
let data: Vec<Vec<f64>> = (0..n)
|
||||
.map(|_| vec![rng.random_range(0.0..10.0), rng.random_range(0.0..10.0)])
|
||||
.collect();
|
||||
let targets: Vec<f64> = data.iter().map(|row| row[0]).collect();
|
||||
|
||||
let result = compute_fanova(
|
||||
&data,
|
||||
&targets,
|
||||
&["x".into(), "y".into()],
|
||||
&FanovaConfig::default(),
|
||||
);
|
||||
|
||||
assert_eq!(result.main_effects[0].0, "x");
|
||||
assert!(
|
||||
result.main_effects[0].1 > 0.8,
|
||||
"x importance = {}",
|
||||
result.main_effects[0].1
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn interaction_detection() {
|
||||
// f(x, y) = x * y — both matter and interact
|
||||
let mut rng = StdRng::seed_from_u64(0);
|
||||
let n = 200;
|
||||
let data: Vec<Vec<f64>> = (0..n)
|
||||
.map(|_| vec![rng.random_range(0.0..10.0), rng.random_range(0.0..10.0)])
|
||||
.collect();
|
||||
let targets: Vec<f64> = data.iter().map(|row| row[0] * row[1]).collect();
|
||||
|
||||
let config = FanovaConfig {
|
||||
n_trees: 128,
|
||||
..FanovaConfig::default()
|
||||
};
|
||||
let result = compute_fanova(&data, &targets, &["x".into(), "y".into()], &config);
|
||||
|
||||
assert!(
|
||||
!result.interactions.is_empty(),
|
||||
"should detect x*y interaction"
|
||||
);
|
||||
assert!(
|
||||
result.interactions[0].1 > 0.05,
|
||||
"interaction importance = {}",
|
||||
result.interactions[0].1
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn variance_computation() {
|
||||
assert!((variance(&[1.0, 2.0, 3.0, 4.0, 5.0]) - 2.0).abs() < 1e-10);
|
||||
assert!(variance(&[5.0, 5.0, 5.0]).abs() < 1e-10);
|
||||
assert!(variance(&[]).abs() < 1e-10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn three_params_one_dominant() {
|
||||
// f(x, y, z) = 3*x + 0.1*y + 0*z
|
||||
let mut rng = StdRng::seed_from_u64(7);
|
||||
let n = 150;
|
||||
let data: Vec<Vec<f64>> = (0..n)
|
||||
.map(|_| {
|
||||
vec![
|
||||
rng.random_range(0.0..10.0),
|
||||
rng.random_range(0.0..10.0),
|
||||
rng.random_range(0.0..10.0),
|
||||
]
|
||||
})
|
||||
.collect();
|
||||
let targets: Vec<f64> = data.iter().map(|r| 3.0 * r[0] + 0.1 * r[1]).collect();
|
||||
|
||||
let result = compute_fanova(
|
||||
&data,
|
||||
&targets,
|
||||
&["x".into(), "y".into(), "z".into()],
|
||||
&FanovaConfig::default(),
|
||||
);
|
||||
|
||||
// x should be the most important
|
||||
assert_eq!(result.main_effects[0].0, "x");
|
||||
assert!(result.main_effects[0].1 > 0.5);
|
||||
|
||||
// z should have near-zero importance
|
||||
let z_imp = result
|
||||
.main_effects
|
||||
.iter()
|
||||
.find(|(name, _)| name == "z")
|
||||
.map_or(0.0, |(_, v)| *v);
|
||||
assert!(z_imp < 0.1, "z importance = {z_imp}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn importances_sum_to_one() {
|
||||
let mut rng = StdRng::seed_from_u64(3);
|
||||
let n = 100;
|
||||
let data: Vec<Vec<f64>> = (0..n)
|
||||
.map(|_| vec![rng.random_range(0.0..10.0), rng.random_range(0.0..10.0)])
|
||||
.collect();
|
||||
let targets: Vec<f64> = data.iter().map(|r| r[0] + r[1]).collect();
|
||||
|
||||
let result = compute_fanova(
|
||||
&data,
|
||||
&targets,
|
||||
&["x".into(), "y".into()],
|
||||
&FanovaConfig::default(),
|
||||
);
|
||||
|
||||
let total: f64 = result.main_effects.iter().map(|(_, v)| *v).sum::<f64>()
|
||||
+ result.interactions.iter().map(|(_, v)| *v).sum::<f64>();
|
||||
assert!(
|
||||
(total - 1.0).abs() < 1e-10,
|
||||
"importances should sum to 1.0, got {total}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,93 +0,0 @@
|
||||
//! Parameter importance via Spearman rank correlation.
|
||||
|
||||
/// Assign average ranks to a slice of `f64` values (handles ties).
|
||||
#[allow(clippy::cast_precision_loss, clippy::float_cmp)]
|
||||
pub(crate) fn rank(values: &[f64]) -> Vec<f64> {
|
||||
let n = values.len();
|
||||
let mut indexed: Vec<(usize, f64)> = values.iter().copied().enumerate().collect();
|
||||
indexed.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(core::cmp::Ordering::Equal));
|
||||
|
||||
let mut ranks = vec![0.0; n];
|
||||
let mut i = 0;
|
||||
while i < n {
|
||||
// Find the run of tied values.
|
||||
let mut j = i + 1;
|
||||
while j < n && indexed[j].1 == indexed[i].1 {
|
||||
j += 1;
|
||||
}
|
||||
// Average rank for the tie group (1-based ranks).
|
||||
let avg = (i + 1..=j).sum::<usize>() as f64 / (j - i) as f64;
|
||||
for item in &indexed[i..j] {
|
||||
ranks[item.0] = avg;
|
||||
}
|
||||
i = j;
|
||||
}
|
||||
ranks
|
||||
}
|
||||
|
||||
/// Pearson correlation coefficient on two equal-length slices.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
fn pearson(x: &[f64], y: &[f64]) -> f64 {
|
||||
let n = x.len() as f64;
|
||||
let mean_x = x.iter().sum::<f64>() / n;
|
||||
let mean_y = y.iter().sum::<f64>() / n;
|
||||
|
||||
let mut cov = 0.0;
|
||||
let mut var_x = 0.0;
|
||||
let mut var_y = 0.0;
|
||||
for (xi, yi) in x.iter().zip(y.iter()) {
|
||||
let dx = xi - mean_x;
|
||||
let dy = yi - mean_y;
|
||||
cov += dx * dy;
|
||||
var_x += dx * dx;
|
||||
var_y += dy * dy;
|
||||
}
|
||||
|
||||
let denom = (var_x * var_y).sqrt();
|
||||
if denom == 0.0 { 0.0 } else { cov / denom }
|
||||
}
|
||||
|
||||
/// Spearman rank correlation (Pearson on ranks).
|
||||
pub(crate) fn spearman(x: &[f64], y: &[f64]) -> f64 {
|
||||
pearson(&rank(x), &rank(y))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn rank_no_ties() {
|
||||
let ranks = rank(&[30.0, 10.0, 20.0]);
|
||||
assert_eq!(ranks, vec![3.0, 1.0, 2.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rank_with_ties() {
|
||||
let ranks = rank(&[10.0, 20.0, 20.0, 30.0]);
|
||||
assert_eq!(ranks, vec![1.0, 2.5, 2.5, 4.0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn perfect_positive_correlation() {
|
||||
let x = vec![1.0, 2.0, 3.0, 4.0, 5.0];
|
||||
let y = vec![2.0, 4.0, 6.0, 8.0, 10.0];
|
||||
let r = spearman(&x, &y);
|
||||
assert!((r - 1.0).abs() < 1e-10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn perfect_negative_correlation() {
|
||||
let x = vec![1.0, 2.0, 3.0, 4.0, 5.0];
|
||||
let y = vec![10.0, 8.0, 6.0, 4.0, 2.0];
|
||||
let r = spearman(&x, &y);
|
||||
assert!((r + 1.0).abs() < 1e-10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_variance_returns_zero() {
|
||||
let x = vec![5.0, 5.0, 5.0];
|
||||
let y = vec![1.0, 2.0, 3.0];
|
||||
assert!(spearman(&x, &y).abs() < f64::EPSILON);
|
||||
}
|
||||
}
|
||||
@@ -3,9 +3,7 @@
|
||||
//! This module provides a Gaussian kernel density estimator used by the TPE
|
||||
//! sampler to model probability distributions over good and bad trial regions.
|
||||
|
||||
use rand::{Rng, RngExt};
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use rand::Rng;
|
||||
|
||||
/// A Gaussian kernel density estimator for continuous distributions.
|
||||
///
|
||||
@@ -30,7 +28,7 @@ use crate::error::{Error, Result};
|
||||
/// let sample = kde.sample(&mut rng);
|
||||
/// ```
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct KernelDensityEstimator {
|
||||
pub struct KernelDensityEstimator {
|
||||
/// The sample points used to construct the KDE.
|
||||
samples: Vec<f64>,
|
||||
/// The bandwidth (standard deviation) of the Gaussian kernels.
|
||||
@@ -40,45 +38,37 @@ pub(crate) struct KernelDensityEstimator {
|
||||
impl KernelDensityEstimator {
|
||||
/// Creates a new KDE with automatic bandwidth selection using Scott's rule.
|
||||
///
|
||||
/// Scott's rule sets bandwidth = n^(-1/5) * `std_dev`, which works well
|
||||
/// Scott's rule sets bandwidth = n^(-1/5) * std_dev, which works well
|
||||
/// for unimodal distributions close to normal.
|
||||
///
|
||||
/// # Errors
|
||||
/// # Panics
|
||||
///
|
||||
/// Returns `Error::EmptySamples` if `samples` is empty.
|
||||
pub(crate) fn new(samples: Vec<f64>) -> Result<Self> {
|
||||
if samples.is_empty() {
|
||||
return Err(Error::EmptySamples);
|
||||
}
|
||||
/// Panics if `samples` is empty.
|
||||
pub fn new(samples: Vec<f64>) -> Self {
|
||||
assert!(!samples.is_empty(), "KDE requires at least one sample");
|
||||
|
||||
let bandwidth = Self::scotts_rule(&samples);
|
||||
Ok(Self { samples, bandwidth })
|
||||
Self { samples, bandwidth }
|
||||
}
|
||||
|
||||
/// Creates a new KDE with a specified bandwidth.
|
||||
///
|
||||
/// Use this when you want explicit control over the smoothing parameter.
|
||||
///
|
||||
/// # Errors
|
||||
/// # Panics
|
||||
///
|
||||
/// Returns `Error::EmptySamples` if `samples` is empty.
|
||||
/// Returns `Error::InvalidBandwidth` if `bandwidth` is not positive.
|
||||
pub(crate) fn with_bandwidth(samples: Vec<f64>, bandwidth: f64) -> Result<Self> {
|
||||
if samples.is_empty() {
|
||||
return Err(Error::EmptySamples);
|
||||
}
|
||||
if bandwidth <= 0.0 {
|
||||
return Err(Error::InvalidBandwidth(bandwidth));
|
||||
}
|
||||
/// Panics if `samples` is empty or `bandwidth` is not positive.
|
||||
pub(crate) fn with_bandwidth(samples: Vec<f64>, bandwidth: f64) -> Self {
|
||||
assert!(!samples.is_empty(), "KDE requires at least one sample");
|
||||
assert!(bandwidth > 0.0, "Bandwidth must be positive");
|
||||
|
||||
Ok(Self { samples, bandwidth })
|
||||
Self { samples, bandwidth }
|
||||
}
|
||||
|
||||
/// Computes bandwidth using Scott's rule.
|
||||
///
|
||||
/// Scott's rule: h = n^(-1/5) * sigma
|
||||
/// where sigma is the sample standard deviation.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
fn scotts_rule(samples: &[f64]) -> f64 {
|
||||
let n = samples.len() as f64;
|
||||
let std_dev = Self::sample_std_dev(samples);
|
||||
@@ -93,7 +83,6 @@ impl KernelDensityEstimator {
|
||||
}
|
||||
|
||||
/// Computes the sample standard deviation.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
fn sample_std_dev(samples: &[f64]) -> f64 {
|
||||
let n = samples.len() as f64;
|
||||
let mean = samples.iter().sum::<f64>() / n;
|
||||
@@ -106,14 +95,13 @@ impl KernelDensityEstimator {
|
||||
/// The density is computed as the average of Gaussian kernels centered
|
||||
/// at each sample point:
|
||||
///
|
||||
/// f(x) = (1/n) * `sum_i` K((x - `x_i`) / h)
|
||||
/// f(x) = (1/n) * sum_i K((x - x_i) / h)
|
||||
///
|
||||
/// where K is the standard Gaussian kernel and h is the bandwidth.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
pub(crate) fn pdf(&self, x: f64) -> f64 {
|
||||
pub fn pdf(&self, x: f64) -> f64 {
|
||||
let n = self.samples.len() as f64;
|
||||
let inv_bandwidth = 1.0 / self.bandwidth;
|
||||
let normalization = inv_bandwidth / (2.0 * core::f64::consts::PI).sqrt();
|
||||
let normalization = inv_bandwidth / (2.0 * std::f64::consts::PI).sqrt();
|
||||
|
||||
let density: f64 = self
|
||||
.samples
|
||||
@@ -132,7 +120,7 @@ impl KernelDensityEstimator {
|
||||
/// Sampling works by:
|
||||
/// 1. Uniformly selecting one of the kernel centers (samples)
|
||||
/// 2. Adding Gaussian noise with the bandwidth as standard deviation
|
||||
pub(crate) fn sample<R: Rng>(&self, rng: &mut R) -> f64 {
|
||||
pub fn sample<R: Rng>(&self, rng: &mut R) -> f64 {
|
||||
// Select a random sample to center the kernel on
|
||||
let idx = rng.random_range(0..self.samples.len());
|
||||
let center = self.samples[idx];
|
||||
@@ -142,7 +130,7 @@ impl KernelDensityEstimator {
|
||||
let u1: f64 = rng.random();
|
||||
let u2: f64 = rng.random();
|
||||
|
||||
let z = (-2.0 * u1.ln()).sqrt() * (2.0 * core::f64::consts::PI * u2).cos();
|
||||
let z = (-2.0 * u1.ln()).sqrt() * (2.0 * std::f64::consts::PI * u2).cos();
|
||||
center + z * self.bandwidth
|
||||
}
|
||||
|
||||
@@ -160,7 +148,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_kde_pdf_basic() {
|
||||
let samples = vec![0.0, 1.0, 2.0];
|
||||
let kde = KernelDensityEstimator::new(samples).unwrap();
|
||||
let kde = KernelDensityEstimator::new(samples);
|
||||
|
||||
// Density should be positive everywhere
|
||||
assert!(kde.pdf(0.0) > 0.0);
|
||||
@@ -176,17 +164,17 @@ mod tests {
|
||||
#[test]
|
||||
fn test_kde_pdf_integrates_to_one() {
|
||||
let samples = vec![0.0, 1.0, 2.0, 3.0, 4.0];
|
||||
let kde = KernelDensityEstimator::new(samples).unwrap();
|
||||
let kde = KernelDensityEstimator::new(samples);
|
||||
|
||||
// Numerical integration over a wide range
|
||||
let n_points = 10000;
|
||||
let low = -10.0;
|
||||
let high = 15.0;
|
||||
let dx = (high - low) / f64::from(n_points);
|
||||
let dx = (high - low) / n_points as f64;
|
||||
|
||||
let integral: f64 = (0..n_points)
|
||||
.map(|i| {
|
||||
let x = low + (f64::from(i) + 0.5) * dx;
|
||||
let x = low + (i as f64 + 0.5) * dx;
|
||||
kde.pdf(x) * dx
|
||||
})
|
||||
.sum();
|
||||
@@ -201,16 +189,16 @@ mod tests {
|
||||
#[test]
|
||||
fn test_kde_with_bandwidth() {
|
||||
let samples = vec![0.0, 1.0, 2.0];
|
||||
let kde = KernelDensityEstimator::with_bandwidth(samples, 0.5).unwrap();
|
||||
let kde = KernelDensityEstimator::with_bandwidth(samples, 0.5);
|
||||
|
||||
assert!((kde.bandwidth() - 0.5).abs() < f64::EPSILON);
|
||||
assert_eq!(kde.bandwidth(), 0.5);
|
||||
assert!(kde.pdf(1.0) > 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_kde_sample_in_reasonable_range() {
|
||||
let samples = vec![0.0, 1.0, 2.0, 3.0, 4.0];
|
||||
let kde = KernelDensityEstimator::new(samples).unwrap();
|
||||
let kde = KernelDensityEstimator::new(samples);
|
||||
let mut rng = rand::rng();
|
||||
|
||||
// Samples should generally be in a reasonable range around the data
|
||||
@@ -225,7 +213,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_kde_single_sample() {
|
||||
let samples = vec![5.0];
|
||||
let kde = KernelDensityEstimator::new(samples).unwrap();
|
||||
let kde = KernelDensityEstimator::new(samples);
|
||||
|
||||
// Should have positive density near the sample
|
||||
assert!(kde.pdf(5.0) > 0.0);
|
||||
@@ -235,7 +223,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_kde_identical_samples() {
|
||||
let samples = vec![3.0, 3.0, 3.0, 3.0];
|
||||
let kde = KernelDensityEstimator::new(samples).unwrap();
|
||||
let kde = KernelDensityEstimator::new(samples);
|
||||
|
||||
// Should handle degenerate case with identical samples
|
||||
assert!(kde.bandwidth() > 0.0);
|
||||
@@ -245,7 +233,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_scotts_rule_bandwidth() {
|
||||
let samples = vec![0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0];
|
||||
let kde = KernelDensityEstimator::new(samples).unwrap();
|
||||
let kde = KernelDensityEstimator::new(samples);
|
||||
|
||||
// n = 10, n^(-1/5) ≈ 0.631
|
||||
// std_dev ≈ 2.87
|
||||
@@ -258,23 +246,23 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "KDE requires at least one sample")]
|
||||
fn test_kde_empty_samples() {
|
||||
let samples: Vec<f64> = vec![];
|
||||
let result = KernelDensityEstimator::new(samples);
|
||||
assert!(matches!(result, Err(Error::EmptySamples)));
|
||||
KernelDensityEstimator::new(samples);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "Bandwidth must be positive")]
|
||||
fn test_kde_zero_bandwidth() {
|
||||
let samples = vec![1.0, 2.0, 3.0];
|
||||
let result = KernelDensityEstimator::with_bandwidth(samples, 0.0);
|
||||
assert!(matches!(result, Err(Error::InvalidBandwidth(_))));
|
||||
KernelDensityEstimator::with_bandwidth(samples, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "Bandwidth must be positive")]
|
||||
fn test_kde_negative_bandwidth() {
|
||||
let samples = vec![1.0, 2.0, 3.0];
|
||||
let result = KernelDensityEstimator::with_bandwidth(samples, -1.0);
|
||||
assert!(matches!(result, Err(Error::InvalidBandwidth(_))));
|
||||
KernelDensityEstimator::with_bandwidth(samples, -1.0);
|
||||
}
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
//! Kernel Density Estimation for parameter distributions.
|
||||
//!
|
||||
//! This module provides kernel density estimators used by TPE samplers
|
||||
//! to model probability distributions over good and bad trial regions.
|
||||
//!
|
||||
//! - [`univariate`] - Univariate (single-parameter) KDE
|
||||
//! - [`multivariate`] - Multivariate (joint-parameter) KDE for capturing parameter dependencies
|
||||
|
||||
mod multivariate;
|
||||
mod univariate;
|
||||
|
||||
pub(crate) use multivariate::MultivariateKDE;
|
||||
pub(crate) use univariate::KernelDensityEstimator;
|
||||
@@ -1,921 +0,0 @@
|
||||
//! Multivariate Kernel Density Estimation for joint parameter distributions.
|
||||
//!
|
||||
//! This module provides a multivariate kernel density estimator that captures
|
||||
//! dependencies between parameters. Unlike the univariate KDE which models each
|
||||
//! parameter independently, the multivariate KDE models the joint distribution
|
||||
//! to better capture correlations between parameters.
|
||||
|
||||
use rand::{Rng, RngExt};
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
|
||||
/// A multivariate Gaussian kernel density estimator for joint distributions.
|
||||
///
|
||||
/// `MultivariateKDE` estimates a joint probability density function from a set
|
||||
/// of multi-dimensional samples. This is used in multivariate TPE to model
|
||||
/// the joint distributions l(x) and g(x) across multiple parameters simultaneously,
|
||||
/// capturing their dependencies.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```ignore
|
||||
/// use crate::kde::MultivariateKDE;
|
||||
///
|
||||
/// // Create samples: 3 samples with 2 dimensions each
|
||||
/// let samples = vec![
|
||||
/// vec![1.0, 2.0],
|
||||
/// vec![1.5, 2.5],
|
||||
/// vec![2.0, 3.0],
|
||||
/// ];
|
||||
/// let kde = MultivariateKDE::new(samples).unwrap();
|
||||
///
|
||||
/// // Get dimensionality
|
||||
/// assert_eq!(kde.n_dims(), 2);
|
||||
/// ```
|
||||
#[allow(dead_code)] // Fields and methods will be used in subsequent stories (US-003, US-004)
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct MultivariateKDE {
|
||||
/// The sample points used to construct the KDE.
|
||||
/// Each inner Vec is one sample with `n_dims` values.
|
||||
samples: Vec<Vec<f64>>,
|
||||
/// The bandwidth (standard deviation) for each dimension.
|
||||
/// Uses a diagonal bandwidth matrix (independent bandwidths per dimension).
|
||||
bandwidths: Vec<f64>,
|
||||
/// The number of dimensions.
|
||||
n_dims: usize,
|
||||
}
|
||||
|
||||
#[allow(dead_code)] // Methods will be used in subsequent stories (US-003, US-004)
|
||||
impl MultivariateKDE {
|
||||
/// Creates a new multivariate KDE with automatic bandwidth selection using Scott's rule.
|
||||
///
|
||||
/// Scott's rule for multivariate KDE sets bandwidth per dimension as:
|
||||
/// `h_j = n^(-1/(d+4)) * sigma_j`
|
||||
///
|
||||
/// where n is the number of samples, d is the dimensionality, and `sigma_j` is the
|
||||
/// standard deviation of the j-th dimension.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns `Error::EmptySamples` if `samples` is empty.
|
||||
/// Returns `Error::DimensionMismatch` if samples have inconsistent dimensions.
|
||||
/// Returns `Error::ZeroDimensions` if samples have zero dimensions.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
pub(crate) fn new(samples: Vec<Vec<f64>>) -> Result<Self> {
|
||||
if samples.is_empty() {
|
||||
return Err(Error::EmptySamples);
|
||||
}
|
||||
|
||||
let n_dims = samples[0].len();
|
||||
if n_dims == 0 {
|
||||
return Err(Error::ZeroDimensions);
|
||||
}
|
||||
|
||||
// Check all samples have the same dimensionality
|
||||
for (i, sample) in samples.iter().enumerate() {
|
||||
if sample.len() != n_dims {
|
||||
return Err(Error::DimensionMismatch {
|
||||
expected: n_dims,
|
||||
got: sample.len(),
|
||||
sample_index: i,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let bandwidths = Self::scotts_rule_multivariate(&samples, n_dims);
|
||||
|
||||
Ok(Self {
|
||||
samples,
|
||||
bandwidths,
|
||||
n_dims,
|
||||
})
|
||||
}
|
||||
|
||||
/// Creates a new multivariate KDE with specified bandwidths.
|
||||
///
|
||||
/// Use this when you want explicit control over the smoothing parameters.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns `Error::EmptySamples` if `samples` is empty.
|
||||
/// Returns `Error::DimensionMismatch` if samples have inconsistent dimensions.
|
||||
/// Returns `Error::ZeroDimensions` if samples have zero dimensions.
|
||||
/// Returns `Error::BandwidthDimensionMismatch` if bandwidths length doesn't match dimensions.
|
||||
/// Returns `Error::InvalidBandwidth` if any bandwidth is not positive.
|
||||
pub(crate) fn with_bandwidths(samples: Vec<Vec<f64>>, bandwidths: Vec<f64>) -> Result<Self> {
|
||||
if samples.is_empty() {
|
||||
return Err(Error::EmptySamples);
|
||||
}
|
||||
|
||||
let n_dims = samples[0].len();
|
||||
if n_dims == 0 {
|
||||
return Err(Error::ZeroDimensions);
|
||||
}
|
||||
|
||||
// Check all samples have the same dimensionality
|
||||
for (i, sample) in samples.iter().enumerate() {
|
||||
if sample.len() != n_dims {
|
||||
return Err(Error::DimensionMismatch {
|
||||
expected: n_dims,
|
||||
got: sample.len(),
|
||||
sample_index: i,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Check bandwidths length matches dimensions
|
||||
if bandwidths.len() != n_dims {
|
||||
return Err(Error::BandwidthDimensionMismatch {
|
||||
expected: n_dims,
|
||||
got: bandwidths.len(),
|
||||
});
|
||||
}
|
||||
|
||||
// Check all bandwidths are positive
|
||||
for &bw in &bandwidths {
|
||||
if bw <= 0.0 {
|
||||
return Err(Error::InvalidBandwidth(bw));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
samples,
|
||||
bandwidths,
|
||||
n_dims,
|
||||
})
|
||||
}
|
||||
|
||||
/// Computes bandwidths using Scott's rule for multivariate KDE.
|
||||
///
|
||||
/// Scott's rule for d dimensions: `h_j = n^(-1/(d+4)) * sigma_j`
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
fn scotts_rule_multivariate(samples: &[Vec<f64>], n_dims: usize) -> Vec<f64> {
|
||||
let n = samples.len() as f64;
|
||||
let d = n_dims as f64;
|
||||
|
||||
// Scott's rule exponent for multivariate: -1/(d+4)
|
||||
let exponent = -1.0 / (d + 4.0);
|
||||
let scale_factor = n.powf(exponent);
|
||||
|
||||
(0..n_dims)
|
||||
.map(|dim| {
|
||||
let std_dev = Self::dimension_std_dev(samples, dim);
|
||||
// For degenerate case where all samples are identical in this dimension,
|
||||
// use a small positive bandwidth
|
||||
if std_dev < f64::EPSILON {
|
||||
1.0
|
||||
} else {
|
||||
scale_factor * std_dev
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Computes the sample standard deviation for a single dimension.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
fn dimension_std_dev(samples: &[Vec<f64>], dim: usize) -> f64 {
|
||||
let n = samples.len() as f64;
|
||||
let values: Vec<f64> = samples.iter().map(|s| s[dim]).collect();
|
||||
let mean = values.iter().sum::<f64>() / n;
|
||||
let variance = values.iter().map(|x| (x - mean).powi(2)).sum::<f64>() / n;
|
||||
variance.sqrt()
|
||||
}
|
||||
|
||||
/// Returns the number of dimensions.
|
||||
pub(crate) fn n_dims(&self) -> usize {
|
||||
self.n_dims
|
||||
}
|
||||
|
||||
/// Returns the number of samples.
|
||||
#[allow(dead_code)] // Will be used in subsequent stories
|
||||
pub(crate) fn n_samples(&self) -> usize {
|
||||
self.samples.len()
|
||||
}
|
||||
|
||||
/// Returns the bandwidths for each dimension.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn bandwidths(&self) -> &[f64] {
|
||||
&self.bandwidths
|
||||
}
|
||||
|
||||
/// Returns a reference to the samples.
|
||||
#[allow(dead_code)] // Will be used in subsequent stories
|
||||
pub(crate) fn samples(&self) -> &[Vec<f64>] {
|
||||
&self.samples
|
||||
}
|
||||
|
||||
/// Returns the log probability density at point `x`.
|
||||
///
|
||||
/// This computes the log-density for numerical stability, using the formula:
|
||||
///
|
||||
/// `log f(x) = log((1/n) * Σ_i Π_j K_hj((x_j - x_ij) / h_j))`
|
||||
///
|
||||
/// The computation uses the log-sum-exp trick for numerical stability:
|
||||
///
|
||||
/// `log(Σ exp(a_i)) = max(a) + log(Σ exp(a_i - max(a)))`
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `x.len() != self.n_dims`.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
pub(crate) fn log_pdf(&self, x: &[f64]) -> f64 {
|
||||
assert_eq!(
|
||||
x.len(),
|
||||
self.n_dims,
|
||||
"Point dimension {} doesn't match KDE dimension {}",
|
||||
x.len(),
|
||||
self.n_dims
|
||||
);
|
||||
|
||||
let n = self.samples.len() as f64;
|
||||
|
||||
// Precompute log normalization constant for each dimension
|
||||
// For a Gaussian kernel: K_h(z) = (1/(h*sqrt(2*pi))) * exp(-0.5*z^2)
|
||||
// log(K_h(z)) = -log(h) - 0.5*log(2*pi) - 0.5*z^2
|
||||
let log_2pi = (2.0 * core::f64::consts::PI).ln();
|
||||
let log_norm_per_dim: Vec<f64> = self
|
||||
.bandwidths
|
||||
.iter()
|
||||
.map(|&h| -h.ln() - 0.5 * log_2pi)
|
||||
.collect();
|
||||
|
||||
// Compute log of kernel contribution for each sample
|
||||
// log(prod_j K_hj(z_j)) = sum_j log(K_hj(z_j))
|
||||
let log_kernels: Vec<f64> = self
|
||||
.samples
|
||||
.iter()
|
||||
.map(|sample| {
|
||||
let mut log_kernel_sum = 0.0;
|
||||
for j in 0..self.n_dims {
|
||||
let z = (x[j] - sample[j]) / self.bandwidths[j];
|
||||
log_kernel_sum += log_norm_per_dim[j] - 0.5 * z * z;
|
||||
}
|
||||
log_kernel_sum
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Use log-sum-exp trick for numerical stability
|
||||
// log(sum(exp(log_kernels))) = max + log(sum(exp(log_kernels - max)))
|
||||
let max_log_kernel = log_kernels
|
||||
.iter()
|
||||
.copied()
|
||||
.fold(f64::NEG_INFINITY, f64::max);
|
||||
|
||||
// Handle case where all log_kernels are -inf (extremely unlikely point)
|
||||
if max_log_kernel.is_infinite() && max_log_kernel < 0.0 {
|
||||
return f64::NEG_INFINITY;
|
||||
}
|
||||
|
||||
let sum_exp: f64 = log_kernels
|
||||
.iter()
|
||||
.map(|&lk| (lk - max_log_kernel).exp())
|
||||
.sum();
|
||||
|
||||
// log((1/n) * sum) = -log(n) + log(sum)
|
||||
-n.ln() + max_log_kernel + sum_exp.ln()
|
||||
}
|
||||
|
||||
/// Returns the probability density at point `x`.
|
||||
///
|
||||
/// The density is computed as the average of multivariate Gaussian kernels
|
||||
/// centered at each sample point:
|
||||
///
|
||||
/// `f(x) = (1/n) * Σ_i Π_j K_hj((x_j - x_ij) / h_j)`
|
||||
///
|
||||
/// where `K_hj` is the univariate Gaussian kernel with bandwidth `h_j`.
|
||||
///
|
||||
/// This method computes in log-space for numerical stability and then
|
||||
/// exponentiates the result.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `x.len() != self.n_dims`.
|
||||
pub(crate) fn pdf(&self, x: &[f64]) -> f64 {
|
||||
self.log_pdf(x).exp()
|
||||
}
|
||||
|
||||
/// Samples a point from the estimated joint density distribution.
|
||||
///
|
||||
/// Sampling works by:
|
||||
/// 1. Uniformly selecting one of the kernel centers (samples)
|
||||
/// 2. Adding independent Gaussian noise to each dimension with that
|
||||
/// dimension's bandwidth as the standard deviation
|
||||
///
|
||||
/// This is equivalent to sampling from a mixture of multivariate Gaussians
|
||||
/// with diagonal covariance matrices, where each mixture component is
|
||||
/// centered at a sample point.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A `Vec<f64>` of length `n_dims` representing a sample from the KDE.
|
||||
pub(crate) fn sample<R: Rng>(&self, rng: &mut R) -> Vec<f64> {
|
||||
// Select a random sample to center the kernel on
|
||||
let idx = rng.random_range(0..self.samples.len());
|
||||
let center = &self.samples[idx];
|
||||
|
||||
// Add independent Gaussian noise to each dimension
|
||||
// Using Box-Muller transform for Gaussian sampling
|
||||
center
|
||||
.iter()
|
||||
.zip(self.bandwidths.iter())
|
||||
.map(|(¢er_j, &bandwidth_j)| {
|
||||
let u1: f64 = rng.random();
|
||||
let u2: f64 = rng.random();
|
||||
|
||||
// Box-Muller transform: generates standard normal variate
|
||||
let z = (-2.0 * u1.ln()).sqrt() * (2.0 * core::f64::consts::PI * u2).cos();
|
||||
|
||||
// Scale by bandwidth and shift by center
|
||||
center_j + z * bandwidth_j
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_new_basic() {
|
||||
let samples = vec![vec![1.0, 2.0], vec![1.5, 2.5], vec![2.0, 3.0]];
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
|
||||
assert_eq!(kde.n_dims(), 2);
|
||||
assert_eq!(kde.n_samples(), 3);
|
||||
assert_eq!(kde.bandwidths().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_new_single_sample() {
|
||||
let samples = vec![vec![1.0, 2.0, 3.0]];
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
|
||||
assert_eq!(kde.n_dims(), 3);
|
||||
assert_eq!(kde.n_samples(), 1);
|
||||
// With single sample, bandwidths should default to 1.0 (degenerate case)
|
||||
for &bw in kde.bandwidths() {
|
||||
assert!((bw - 1.0).abs() < f64::EPSILON);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_new_single_dimension() {
|
||||
let samples = vec![vec![1.0], vec![2.0], vec![3.0], vec![4.0], vec![5.0]];
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
|
||||
assert_eq!(kde.n_dims(), 1);
|
||||
assert_eq!(kde.n_samples(), 5);
|
||||
assert_eq!(kde.bandwidths().len(), 1);
|
||||
// Bandwidth should be positive
|
||||
assert!(kde.bandwidths()[0] > 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_scotts_rule() {
|
||||
// Create samples with known statistics
|
||||
// 10 samples, 2 dimensions
|
||||
let samples: Vec<Vec<f64>> = (0..10)
|
||||
.map(|i| {
|
||||
let x = f64::from(i);
|
||||
vec![x, x * 2.0] // Second dimension has 2x variance
|
||||
})
|
||||
.collect();
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
|
||||
// Scott's rule: h = n^(-1/(d+4)) * sigma
|
||||
// n=10, d=2: exponent = -1/6 ≈ -0.167
|
||||
// 10^(-1/6) ≈ 0.681
|
||||
// First dim std_dev ≈ 2.87, second ≈ 5.74
|
||||
// Expected bandwidths: ~1.95 and ~3.91
|
||||
|
||||
let bw = kde.bandwidths();
|
||||
assert!(
|
||||
bw[0] > 1.0 && bw[0] < 3.0,
|
||||
"First bandwidth {} unexpected",
|
||||
bw[0]
|
||||
);
|
||||
assert!(
|
||||
bw[1] > 2.0 && bw[1] < 6.0,
|
||||
"Second bandwidth {} unexpected",
|
||||
bw[1]
|
||||
);
|
||||
// Second bandwidth should be approximately 2x the first
|
||||
assert!(
|
||||
(bw[1] / bw[0] - 2.0).abs() < 0.1,
|
||||
"Ratio {} not close to 2",
|
||||
bw[1] / bw[0]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_with_bandwidths() {
|
||||
let samples = vec![vec![1.0, 2.0], vec![3.0, 4.0]];
|
||||
let bandwidths = vec![0.5, 1.0];
|
||||
let kde = MultivariateKDE::with_bandwidths(samples, bandwidths).unwrap();
|
||||
|
||||
assert_eq!(kde.n_dims(), 2);
|
||||
assert!((kde.bandwidths()[0] - 0.5).abs() < f64::EPSILON);
|
||||
assert!((kde.bandwidths()[1] - 1.0).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_empty_samples() {
|
||||
let samples: Vec<Vec<f64>> = vec![];
|
||||
let result = MultivariateKDE::new(samples);
|
||||
assert!(matches!(result, Err(Error::EmptySamples)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_zero_dimensions() {
|
||||
let samples = vec![vec![], vec![]];
|
||||
let result = MultivariateKDE::new(samples);
|
||||
assert!(matches!(result, Err(Error::ZeroDimensions)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_dimension_mismatch() {
|
||||
let samples = vec![vec![1.0, 2.0], vec![3.0]]; // Second sample has wrong dimensions
|
||||
let result = MultivariateKDE::new(samples);
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(Error::DimensionMismatch {
|
||||
expected: 2,
|
||||
got: 1,
|
||||
sample_index: 1
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_with_bandwidths_wrong_length() {
|
||||
let samples = vec![vec![1.0, 2.0], vec![3.0, 4.0]];
|
||||
let bandwidths = vec![0.5]; // Only 1 bandwidth for 2 dimensions
|
||||
let result = MultivariateKDE::with_bandwidths(samples, bandwidths);
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(Error::BandwidthDimensionMismatch {
|
||||
expected: 2,
|
||||
got: 1
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_with_bandwidths_zero() {
|
||||
let samples = vec![vec![1.0, 2.0], vec![3.0, 4.0]];
|
||||
let bandwidths = vec![0.5, 0.0]; // Second bandwidth is zero
|
||||
let result = MultivariateKDE::with_bandwidths(samples, bandwidths);
|
||||
assert!(matches!(result, Err(Error::InvalidBandwidth(bw)) if bw == 0.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_with_bandwidths_negative() {
|
||||
let samples = vec![vec![1.0, 2.0], vec![3.0, 4.0]];
|
||||
let bandwidths = vec![0.5, -1.0]; // Negative bandwidth
|
||||
let result = MultivariateKDE::with_bandwidths(samples, bandwidths);
|
||||
assert!(
|
||||
matches!(result, Err(Error::InvalidBandwidth(bw)) if (bw - (-1.0)).abs() < f64::EPSILON)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_identical_samples() {
|
||||
// All samples identical - should handle degenerate case
|
||||
let samples = vec![vec![5.0, 10.0], vec![5.0, 10.0], vec![5.0, 10.0]];
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
|
||||
// Bandwidths should default to 1.0 for degenerate dimensions
|
||||
for &bw in kde.bandwidths() {
|
||||
assert!((bw - 1.0).abs() < f64::EPSILON);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_high_dimensional() {
|
||||
// Test with higher dimensions
|
||||
let samples: Vec<Vec<f64>> = (0..20)
|
||||
.map(|i| {
|
||||
let x = f64::from(i);
|
||||
vec![x, x * 0.5, x * 2.0, x * 0.1, x * 10.0]
|
||||
})
|
||||
.collect();
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
|
||||
assert_eq!(kde.n_dims(), 5);
|
||||
assert_eq!(kde.n_samples(), 20);
|
||||
assert_eq!(kde.bandwidths().len(), 5);
|
||||
|
||||
// All bandwidths should be positive
|
||||
for &bw in kde.bandwidths() {
|
||||
assert!(bw > 0.0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_samples_accessor() {
|
||||
let samples = vec![vec![1.0, 2.0], vec![3.0, 4.0]];
|
||||
let kde = MultivariateKDE::new(samples.clone()).unwrap();
|
||||
|
||||
assert_eq!(kde.samples(), &samples);
|
||||
}
|
||||
|
||||
// ==================== PDF Tests ====================
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_pdf_basic() {
|
||||
let samples = vec![vec![0.0, 0.0], vec![1.0, 1.0], vec![2.0, 2.0]];
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
|
||||
// Density should be positive everywhere
|
||||
assert!(kde.pdf(&[0.0, 0.0]) > 0.0);
|
||||
assert!(kde.pdf(&[1.0, 1.0]) > 0.0);
|
||||
assert!(kde.pdf(&[2.0, 2.0]) > 0.0);
|
||||
assert!(kde.pdf(&[0.5, 0.5]) > 0.0);
|
||||
|
||||
// Density should be higher near sample points
|
||||
let near_density = kde.pdf(&[1.0, 1.0]);
|
||||
let far_density = kde.pdf(&[10.0, 10.0]);
|
||||
assert!(near_density > far_density);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_pdf_with_custom_bandwidths() {
|
||||
let samples = vec![vec![0.0, 0.0], vec![1.0, 1.0]];
|
||||
let bandwidths = vec![0.5, 0.5];
|
||||
let kde = MultivariateKDE::with_bandwidths(samples, bandwidths).unwrap();
|
||||
|
||||
// Density should be positive
|
||||
assert!(kde.pdf(&[0.5, 0.5]) > 0.0);
|
||||
assert!(kde.pdf(&[0.0, 0.0]) > 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_pdf_single_sample() {
|
||||
let samples = vec![vec![5.0, 10.0]];
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
|
||||
// Should have positive density near the sample
|
||||
assert!(kde.pdf(&[5.0, 10.0]) > 0.0);
|
||||
assert!(kde.pdf(&[4.5, 9.5]) > 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_log_pdf_consistency() {
|
||||
let samples = vec![vec![0.0, 0.0], vec![1.0, 1.0], vec![2.0, 2.0]];
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
|
||||
// log_pdf and pdf should be consistent: exp(log_pdf(x)) == pdf(x)
|
||||
let test_points = vec![
|
||||
vec![0.0, 0.0],
|
||||
vec![1.0, 1.0],
|
||||
vec![0.5, 0.5],
|
||||
vec![3.0, 3.0],
|
||||
];
|
||||
|
||||
for point in test_points {
|
||||
let log_p = kde.log_pdf(&point);
|
||||
let p = kde.pdf(&point);
|
||||
let p_from_log = log_p.exp();
|
||||
assert!(
|
||||
(p - p_from_log).abs() < 1e-10,
|
||||
"pdf={p}, exp(log_pdf)={p_from_log}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_pdf_integrates_to_one_1d() {
|
||||
// Test with 1D case first (should match univariate KDE behavior)
|
||||
let samples = vec![vec![0.0], vec![1.0], vec![2.0], vec![3.0], vec![4.0]];
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
|
||||
// Numerical integration over a wide range
|
||||
let n_points = 1000;
|
||||
let low = -10.0;
|
||||
let high = 15.0;
|
||||
let dx = (high - low) / f64::from(n_points);
|
||||
|
||||
let integral: f64 = (0..n_points)
|
||||
.map(|i| {
|
||||
let x = low + (f64::from(i) + 0.5) * dx;
|
||||
kde.pdf(&[x]) * dx
|
||||
})
|
||||
.sum();
|
||||
|
||||
// Should be approximately 1.0 (within numerical error)
|
||||
assert!(
|
||||
(integral - 1.0).abs() < 0.02,
|
||||
"1D integral = {integral}, expected ~1.0"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_pdf_integrates_to_one_2d() {
|
||||
// Test with 2D case
|
||||
let samples = vec![
|
||||
vec![0.0, 0.0],
|
||||
vec![1.0, 0.0],
|
||||
vec![0.0, 1.0],
|
||||
vec![1.0, 1.0],
|
||||
vec![0.5, 0.5],
|
||||
];
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
|
||||
// Numerical integration over a 2D grid
|
||||
let n_points = 100; // 100x100 = 10000 points
|
||||
let low = -5.0;
|
||||
let high = 6.0;
|
||||
let dx = (high - low) / f64::from(n_points);
|
||||
|
||||
let mut integral = 0.0;
|
||||
for i in 0..n_points {
|
||||
for j in 0..n_points {
|
||||
let x = low + (f64::from(i) + 0.5) * dx;
|
||||
let y = low + (f64::from(j) + 0.5) * dx;
|
||||
integral += kde.pdf(&[x, y]) * dx * dx;
|
||||
}
|
||||
}
|
||||
|
||||
// Should be approximately 1.0 (within numerical error)
|
||||
// 2D integration has more error, so use larger tolerance
|
||||
assert!(
|
||||
(integral - 1.0).abs() < 0.05,
|
||||
"2D integral = {integral}, expected ~1.0"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_pdf_symmetry() {
|
||||
// With symmetric samples, PDF should be symmetric
|
||||
let samples = vec![
|
||||
vec![1.0, 0.0],
|
||||
vec![-1.0, 0.0],
|
||||
vec![0.0, 1.0],
|
||||
vec![0.0, -1.0],
|
||||
];
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
|
||||
// Density at symmetric points should be equal
|
||||
let d1 = kde.pdf(&[0.5, 0.0]);
|
||||
let d2 = kde.pdf(&[-0.5, 0.0]);
|
||||
assert!(
|
||||
(d1 - d2).abs() < 1e-10,
|
||||
"Symmetric points have different densities: {d1} vs {d2}"
|
||||
);
|
||||
|
||||
let d3 = kde.pdf(&[0.0, 0.5]);
|
||||
let d4 = kde.pdf(&[0.0, -0.5]);
|
||||
assert!(
|
||||
(d3 - d4).abs() < 1e-10,
|
||||
"Symmetric points have different densities: {d3} vs {d4}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_pdf_high_dimensional() {
|
||||
// Test with higher dimensions
|
||||
let samples: Vec<Vec<f64>> = (0..10)
|
||||
.map(|i| {
|
||||
let x = f64::from(i) * 0.1;
|
||||
vec![x, x, x, x, x] // 5D
|
||||
})
|
||||
.collect();
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
|
||||
// Density should be positive
|
||||
assert!(kde.pdf(&[0.5, 0.5, 0.5, 0.5, 0.5]) > 0.0);
|
||||
assert!(kde.pdf(&[0.0, 0.0, 0.0, 0.0, 0.0]) > 0.0);
|
||||
|
||||
// Log PDF should be finite
|
||||
let log_p = kde.log_pdf(&[0.5, 0.5, 0.5, 0.5, 0.5]);
|
||||
assert!(log_p.is_finite());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_pdf_numerical_stability() {
|
||||
// Test numerical stability with points far from samples
|
||||
let samples = vec![vec![0.0, 0.0], vec![1.0, 1.0]];
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
|
||||
// Very far point should have very small but non-negative density
|
||||
let far_pdf = kde.pdf(&[100.0, 100.0]);
|
||||
assert!(far_pdf >= 0.0);
|
||||
assert!(far_pdf.is_finite() || far_pdf == 0.0);
|
||||
|
||||
// Log PDF should be finite (or -inf for zero density)
|
||||
let far_log_pdf = kde.log_pdf(&[100.0, 100.0]);
|
||||
assert!(far_log_pdf.is_finite() || far_log_pdf.is_infinite());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "Point dimension")]
|
||||
fn test_multivariate_kde_pdf_wrong_dimension() {
|
||||
let samples = vec![vec![0.0, 0.0], vec![1.0, 1.0]];
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
|
||||
// Should panic with wrong dimension
|
||||
let _ = kde.pdf(&[0.0]); // Only 1 value for 2D KDE
|
||||
}
|
||||
|
||||
// ==================== Sampling Tests ====================
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_sample_basic() {
|
||||
let samples = vec![vec![0.0, 0.0], vec![1.0, 1.0], vec![2.0, 2.0]];
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
let mut rng = rand::rng();
|
||||
|
||||
// Sample should have correct dimensionality
|
||||
let sample = kde.sample(&mut rng);
|
||||
assert_eq!(sample.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_sample_in_reasonable_range() {
|
||||
let samples = vec![
|
||||
vec![0.0, 0.0],
|
||||
vec![1.0, 1.0],
|
||||
vec![2.0, 2.0],
|
||||
vec![3.0, 3.0],
|
||||
vec![4.0, 4.0],
|
||||
];
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
let mut rng = rand::rng();
|
||||
|
||||
// Samples should generally be in a reasonable range around the data
|
||||
for _ in 0..100 {
|
||||
let s = kde.sample(&mut rng);
|
||||
// With high probability, samples should be within a few bandwidths
|
||||
// of the data range. Use a generous range to avoid flaky tests.
|
||||
assert!(
|
||||
s[0] > -10.0 && s[0] < 15.0,
|
||||
"Sample dimension 0: {} outside expected range",
|
||||
s[0]
|
||||
);
|
||||
assert!(
|
||||
s[1] > -10.0 && s[1] < 15.0,
|
||||
"Sample dimension 1: {} outside expected range",
|
||||
s[1]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_sample_single_sample() {
|
||||
// When KDE has only one sample, all samples should be centered around it
|
||||
let samples = vec![vec![5.0, 10.0]];
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
let mut rng = rand::rng();
|
||||
|
||||
// Generate many samples and check they cluster around (5.0, 10.0)
|
||||
let n_samples = 100;
|
||||
let mut sum_x = 0.0;
|
||||
let mut sum_y = 0.0;
|
||||
for _ in 0..n_samples {
|
||||
let s = kde.sample(&mut rng);
|
||||
sum_x += s[0];
|
||||
sum_y += s[1];
|
||||
}
|
||||
|
||||
let mean_x = sum_x / f64::from(n_samples);
|
||||
let mean_y = sum_y / f64::from(n_samples);
|
||||
|
||||
// Mean should be close to the single sample point
|
||||
// With 100 samples and bandwidth=1.0, we expect mean within ~0.3 of center
|
||||
assert!(
|
||||
(mean_x - 5.0).abs() < 1.0,
|
||||
"Mean x={mean_x}, expected close to 5.0"
|
||||
);
|
||||
assert!(
|
||||
(mean_y - 10.0).abs() < 1.0,
|
||||
"Mean y={mean_y}, expected close to 10.0"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_sample_high_dimensional() {
|
||||
// Test sampling in higher dimensions
|
||||
let samples: Vec<Vec<f64>> = (0..10)
|
||||
.map(|i| {
|
||||
let x = f64::from(i) * 0.5;
|
||||
vec![x, x * 2.0, x * 0.5, x + 1.0, x - 1.0] // 5D
|
||||
})
|
||||
.collect();
|
||||
let kde = MultivariateKDE::new(samples).unwrap();
|
||||
let mut rng = rand::rng();
|
||||
|
||||
// Sample should have correct dimensionality
|
||||
for _ in 0..50 {
|
||||
let sample = kde.sample(&mut rng);
|
||||
assert_eq!(sample.len(), 5);
|
||||
// All values should be finite
|
||||
for &val in &sample {
|
||||
assert!(val.is_finite(), "Sample value is not finite: {val}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
fn test_multivariate_kde_sample_respects_bandwidth() {
|
||||
// Create samples all at origin with large custom bandwidth
|
||||
let data = vec![vec![0.0, 0.0], vec![0.0, 0.0], vec![0.0, 0.0]];
|
||||
let bandwidths = vec![0.1, 10.0]; // Small bandwidth in x, large in y
|
||||
let kde = MultivariateKDE::with_bandwidths(data, bandwidths).unwrap();
|
||||
let mut rng = rand::rng();
|
||||
|
||||
// Generate samples and check variance in each dimension
|
||||
let n_samples = 1000;
|
||||
let mut values_x: Vec<f64> = Vec::with_capacity(n_samples);
|
||||
let mut values_y: Vec<f64> = Vec::with_capacity(n_samples);
|
||||
|
||||
for _ in 0..n_samples {
|
||||
let s = kde.sample(&mut rng);
|
||||
values_x.push(s[0]);
|
||||
values_y.push(s[1]);
|
||||
}
|
||||
|
||||
// Compute sample variances
|
||||
let n = n_samples as f64;
|
||||
let mean_x: f64 = values_x.iter().sum::<f64>() / n;
|
||||
let mean_y: f64 = values_y.iter().sum::<f64>() / n;
|
||||
|
||||
let var_x: f64 = values_x.iter().map(|x| (x - mean_x).powi(2)).sum::<f64>() / n;
|
||||
let var_y: f64 = values_y.iter().map(|y| (y - mean_y).powi(2)).sum::<f64>() / n;
|
||||
|
||||
// Variance should be approximately bandwidth^2
|
||||
// x: bandwidth=0.1, expected variance ~0.01
|
||||
// y: bandwidth=10.0, expected variance ~100.0
|
||||
assert!(
|
||||
var_x < 0.05,
|
||||
"X variance {var_x} too large for bandwidth 0.1"
|
||||
);
|
||||
assert!(
|
||||
var_y > 50.0 && var_y < 200.0,
|
||||
"Y variance {var_y} unexpected for bandwidth 10.0"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_sample_distribution_shape() {
|
||||
// Create samples along a diagonal line
|
||||
let data = vec![
|
||||
vec![0.0, 0.0],
|
||||
vec![1.0, 1.0],
|
||||
vec![2.0, 2.0],
|
||||
vec![3.0, 3.0],
|
||||
vec![4.0, 4.0],
|
||||
];
|
||||
let kde = MultivariateKDE::new(data).unwrap();
|
||||
let mut rng = rand::rng();
|
||||
|
||||
// Sample many points and verify the mean is near the center
|
||||
let n_samples = 500;
|
||||
let mut sum = [0.0, 0.0];
|
||||
|
||||
for _ in 0..n_samples {
|
||||
let s = kde.sample(&mut rng);
|
||||
sum[0] += s[0];
|
||||
sum[1] += s[1];
|
||||
}
|
||||
|
||||
let mean_x = sum[0] / f64::from(n_samples);
|
||||
let mean_y = sum[1] / f64::from(n_samples);
|
||||
|
||||
// Mean should be close to (2.0, 2.0) - the center of the samples
|
||||
assert!(
|
||||
(mean_x - 2.0).abs() < 0.5,
|
||||
"Mean x={mean_x}, expected close to 2.0"
|
||||
);
|
||||
assert!(
|
||||
(mean_y - 2.0).abs() < 0.5,
|
||||
"Mean y={mean_y}, expected close to 2.0"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_kde_sample_deterministic_with_seeded_rng() {
|
||||
use rand::SeedableRng;
|
||||
|
||||
let data = vec![vec![1.0, 2.0], vec![3.0, 4.0]];
|
||||
let kde = MultivariateKDE::new(data).unwrap();
|
||||
|
||||
// Use a seeded RNG for reproducibility
|
||||
let mut rng1 = rand::rngs::StdRng::seed_from_u64(42);
|
||||
let mut rng2 = rand::rngs::StdRng::seed_from_u64(42);
|
||||
|
||||
// Same seed should produce same samples
|
||||
let result1 = kde.sample(&mut rng1);
|
||||
let result2 = kde.sample(&mut rng2);
|
||||
|
||||
assert!(
|
||||
(result1[0] - result2[0]).abs() < f64::EPSILON,
|
||||
"Samples with same seed differ in dimension 0"
|
||||
);
|
||||
assert!(
|
||||
(result1[1] - result2[1]).abs() < f64::EPSILON,
|
||||
"Samples with same seed differ in dimension 1"
|
||||
);
|
||||
}
|
||||
}
|
||||
+37
-215
@@ -1,29 +1,7 @@
|
||||
#![forbid(unsafe_code)]
|
||||
#![deny(clippy::all)]
|
||||
#![deny(unreachable_pub)]
|
||||
#![deny(clippy::correctness)]
|
||||
#![deny(clippy::suspicious)]
|
||||
#![deny(clippy::style)]
|
||||
#![deny(clippy::complexity)]
|
||||
#![deny(clippy::perf)]
|
||||
#![deny(clippy::pedantic)]
|
||||
#![deny(clippy::std_instead_of_core)]
|
||||
|
||||
//! A black-box optimization library with multiple sampling strategies.
|
||||
//! A Tree-Parzen Estimator (TPE) library for black-box optimization.
|
||||
//!
|
||||
//! This library provides an Optuna-like API for hyperparameter optimization
|
||||
//! with support for multiple sampling algorithms:
|
||||
//!
|
||||
//! - **Random Search** - Simple random sampling for baseline comparisons
|
||||
//! - **TPE (Tree-Parzen Estimator)** - Bayesian optimization for efficient search
|
||||
//! - **Grid Search** - Exhaustive search over a specified parameter grid
|
||||
//! - **Sobol (QMC)** - Quasi-random sampling for better space coverage (requires `sobol` feature)
|
||||
//! - **CMA-ES** - Covariance Matrix Adaptation Evolution Strategy for continuous optimization (requires `cma-es` feature)
|
||||
//! - **BOHB** - Bayesian Optimization + `HyperBand` for budget-aware TPE sampling
|
||||
//! - **NSGA-II** - Non-dominated Sorting Genetic Algorithm II for multi-objective optimization
|
||||
//! - **MOTPE** - Multi-Objective Tree-Parzen Estimator for Bayesian multi-objective optimization
|
||||
//!
|
||||
//! Additional features include:
|
||||
//! using the Tree-Parzen Estimator algorithm. It supports:
|
||||
//!
|
||||
//! - Float, integer, and categorical parameter types
|
||||
//! - Log-scale and stepped parameter sampling
|
||||
@@ -33,26 +11,23 @@
|
||||
//! # Quick Start
|
||||
//!
|
||||
//! ```
|
||||
//! use optimizer::prelude::*;
|
||||
//! use optimizer::{Direction, Study, TpeSampler};
|
||||
//!
|
||||
//! // Create a study with TPE sampler
|
||||
//! let sampler = TpeSampler::builder().seed(42).build().unwrap();
|
||||
//! let sampler = TpeSampler::builder().seed(42).build();
|
||||
//! let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
//!
|
||||
//! // Define parameter search space
|
||||
//! let x = FloatParam::new(-10.0, 10.0).name("x");
|
||||
//!
|
||||
//! // Optimize x^2 for 20 trials
|
||||
//! study
|
||||
//! .optimize(20, |trial| {
|
||||
//! let x_val = x.suggest(trial)?;
|
||||
//! Ok::<_, Error>(x_val * x_val)
|
||||
//! .optimize_with_sampler(20, |trial| {
|
||||
//! let x = trial.suggest_float("x", -10.0, 10.0)?;
|
||||
//! Ok::<_, optimizer::TpeError>(x * x)
|
||||
//! })
|
||||
//! .unwrap();
|
||||
//!
|
||||
//! // Get the best result
|
||||
//! let best = study.best_trial().unwrap();
|
||||
//! println!("x = {}", best.get(&x).unwrap());
|
||||
//! println!("Best value: {} at x={:?}", best.value, best.params);
|
||||
//! ```
|
||||
//!
|
||||
//! # Creating a Study
|
||||
@@ -60,9 +35,7 @@
|
||||
//! A [`Study`] manages optimization trials. Create one with an optimization direction:
|
||||
//!
|
||||
//! ```
|
||||
//! use optimizer::sampler::random::RandomSampler;
|
||||
//! use optimizer::sampler::tpe::TpeSampler;
|
||||
//! use optimizer::{Direction, Study};
|
||||
//! use optimizer::{Direction, RandomSampler, Study, TpeSampler};
|
||||
//!
|
||||
//! // Minimize with default random sampler
|
||||
//! let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
@@ -76,82 +49,47 @@
|
||||
//!
|
||||
//! # Suggesting Parameters
|
||||
//!
|
||||
//! Within the objective function, use parameter types to suggest values:
|
||||
//! Within the objective function, use [`Trial`] to suggest parameter values:
|
||||
//!
|
||||
//! ```
|
||||
//! use optimizer::parameter::{BoolParam, CategoricalParam, FloatParam, IntParam, Parameter};
|
||||
//! use optimizer::{Direction, Study};
|
||||
//!
|
||||
//! let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
//!
|
||||
//! // Define parameter search spaces
|
||||
//! let x_param = FloatParam::new(0.0, 1.0);
|
||||
//! let lr_param = FloatParam::new(1e-5, 1e-1).log_scale();
|
||||
//! let step_param = FloatParam::new(0.0, 1.0).step(0.1);
|
||||
//! let n_param = IntParam::new(1, 10);
|
||||
//! let batch_param = IntParam::new(16, 256).log_scale();
|
||||
//! let units_param = IntParam::new(32, 512).step(32);
|
||||
//! let flag_param = BoolParam::new();
|
||||
//! let optimizer_param = CategoricalParam::new(vec!["sgd", "adam", "rmsprop"]);
|
||||
//!
|
||||
//! study
|
||||
//! .optimize(10, |trial| {
|
||||
//! let x = x_param.suggest(trial)?;
|
||||
//! let lr = lr_param.suggest(trial)?;
|
||||
//! let step = step_param.suggest(trial)?;
|
||||
//! let n = n_param.suggest(trial)?;
|
||||
//! let batch = batch_param.suggest(trial)?;
|
||||
//! let units = units_param.suggest(trial)?;
|
||||
//! let flag = flag_param.suggest(trial)?;
|
||||
//! let optimizer = optimizer_param.suggest(trial)?;
|
||||
//! // Float parameters
|
||||
//! let x = trial.suggest_float("x", 0.0, 1.0)?;
|
||||
//! let lr = trial.suggest_float_log("learning_rate", 1e-5, 1e-1)?;
|
||||
//! let step = trial.suggest_float_step("step", 0.0, 1.0, 0.1)?;
|
||||
//!
|
||||
//! Ok::<_, optimizer::Error>(x * n as f64)
|
||||
//! // Integer parameters
|
||||
//! let n = trial.suggest_int("n_layers", 1, 10)?;
|
||||
//! let batch = trial.suggest_int_log("batch_size", 16, 256)?;
|
||||
//! let units = trial.suggest_int_step("units", 32, 512, 32)?;
|
||||
//!
|
||||
//! // Categorical parameters
|
||||
//! let optimizer = trial.suggest_categorical("optimizer", &["sgd", "adam", "rmsprop"])?;
|
||||
//!
|
||||
//! // Return objective value
|
||||
//! Ok::<_, optimizer::TpeError>(x * n as f64)
|
||||
//! })
|
||||
//! .unwrap();
|
||||
//! ```
|
||||
//!
|
||||
//! # Available Samplers
|
||||
//! # Configuring TPE
|
||||
//!
|
||||
//! ## Random Search
|
||||
//!
|
||||
//! The simplest sampling strategy, useful for baselines:
|
||||
//! The [`TpeSampler`] can be configured using the builder pattern:
|
||||
//!
|
||||
//! ```
|
||||
//! use optimizer::sampler::random::RandomSampler;
|
||||
//! use optimizer::{Direction, Study};
|
||||
//!
|
||||
//! let study: Study<f64> = Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(42));
|
||||
//! ```
|
||||
//!
|
||||
//! ## TPE (Tree-Parzen Estimator)
|
||||
//!
|
||||
//! Bayesian optimization that learns from previous trials:
|
||||
//!
|
||||
//! ```
|
||||
//! use optimizer::sampler::tpe::TpeSampler;
|
||||
//! use optimizer::TpeSampler;
|
||||
//!
|
||||
//! let sampler = TpeSampler::builder()
|
||||
//! .gamma(0.15) // Quantile for good/bad split
|
||||
//! .n_startup_trials(20) // Random trials before TPE
|
||||
//! .n_ei_candidates(32) // Candidates to evaluate
|
||||
//! .seed(42) // Reproducibility
|
||||
//! .build()
|
||||
//! .unwrap();
|
||||
//! ```
|
||||
//!
|
||||
//! ## Grid Search
|
||||
//!
|
||||
//! Exhaustive search over a discretized parameter space:
|
||||
//!
|
||||
//! ```
|
||||
//! use optimizer::sampler::grid::GridSearchSampler;
|
||||
//! use optimizer::{Direction, Study};
|
||||
//!
|
||||
//! let sampler = GridSearchSampler::builder()
|
||||
//! .n_points_per_param(10) // Points per parameter dimension
|
||||
//! .build();
|
||||
//!
|
||||
//! let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
//! ```
|
||||
//!
|
||||
//! # Async and Parallel Optimization
|
||||
@@ -160,151 +98,35 @@
|
||||
//!
|
||||
//! ```ignore
|
||||
//! use optimizer::{Study, Direction};
|
||||
//! use optimizer::parameter::{FloatParam, Parameter};
|
||||
//!
|
||||
//! let x_param = FloatParam::new(0.0, 1.0);
|
||||
//!
|
||||
//! // Sequential async
|
||||
//! study.optimize_async(10, |mut trial| {
|
||||
//! let x_param = x_param.clone();
|
||||
//! async move {
|
||||
//! let x = x_param.suggest(&mut trial)?;
|
||||
//! Ok((trial, x * x))
|
||||
//! }
|
||||
//! study.optimize_async(10, |mut trial| async move {
|
||||
//! let x = trial.suggest_float("x", 0.0, 1.0)?;
|
||||
//! Ok((trial, x * x))
|
||||
//! }).await?;
|
||||
//!
|
||||
//! // Parallel with bounded concurrency
|
||||
//! study.optimize_parallel(10, 4, |mut trial| {
|
||||
//! let x_param = x_param.clone();
|
||||
//! async move {
|
||||
//! let x = x_param.suggest(&mut trial)?;
|
||||
//! Ok((trial, x * x))
|
||||
//! }
|
||||
//! study.optimize_parallel(10, 4, |mut trial| async move {
|
||||
//! let x = trial.suggest_float("x", 0.0, 1.0)?;
|
||||
//! Ok((trial, x * x))
|
||||
//! }).await?;
|
||||
//! ```
|
||||
//!
|
||||
//! # Feature Flags
|
||||
//!
|
||||
//! - `async`: Enable async optimization methods (requires tokio)
|
||||
//! - `derive`: Enable `#[derive(Categorical)]` for enum parameters
|
||||
//! - `serde`: Enable `Serialize`/`Deserialize` on public types and `Study::save()`/`Study::load()`
|
||||
//! - `sobol`: Enable the Sobol quasi-random sampler for better space coverage
|
||||
//! - `cma-es`: Enable the CMA-ES sampler for continuous optimization
|
||||
//! - `visualization`: Generate self-contained HTML reports with interactive Plotly.js charts
|
||||
//! - `tracing`: Emit structured log events via the [`tracing`](https://docs.rs/tracing) crate at key optimization points
|
||||
|
||||
/// Emit a `tracing::info!` event when the `tracing` feature is enabled.
|
||||
/// No-op otherwise.
|
||||
#[cfg(feature = "tracing")]
|
||||
macro_rules! trace_info {
|
||||
($($arg:tt)*) => { tracing::info!($($arg)*) };
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "tracing"))]
|
||||
macro_rules! trace_info {
|
||||
($($arg:tt)*) => {};
|
||||
}
|
||||
|
||||
/// Emit a `tracing::debug!` event when the `tracing` feature is enabled.
|
||||
/// No-op otherwise.
|
||||
#[cfg(feature = "tracing")]
|
||||
macro_rules! trace_debug {
|
||||
($($arg:tt)*) => { tracing::debug!($($arg)*) };
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "tracing"))]
|
||||
macro_rules! trace_debug {
|
||||
($($arg:tt)*) => {};
|
||||
}
|
||||
|
||||
mod distribution;
|
||||
mod error;
|
||||
#[cfg(feature = "fanova")]
|
||||
mod fanova;
|
||||
mod importance;
|
||||
mod kde;
|
||||
pub mod multi_objective;
|
||||
mod param;
|
||||
pub mod parameter;
|
||||
pub mod pareto;
|
||||
pub mod pruner;
|
||||
pub mod sampler;
|
||||
mod sampler;
|
||||
mod study;
|
||||
mod trial;
|
||||
mod types;
|
||||
#[cfg(feature = "visualization")]
|
||||
mod visualization;
|
||||
|
||||
pub use error::{Error, Result, TrialPruned};
|
||||
#[cfg(feature = "fanova")]
|
||||
pub use fanova::{FanovaConfig, FanovaResult};
|
||||
pub use multi_objective::{MultiObjectiveSampler, MultiObjectiveStudy, MultiObjectiveTrial};
|
||||
#[cfg(feature = "derive")]
|
||||
pub use optimizer_derive::Categorical;
|
||||
pub use param::ParamValue;
|
||||
pub use parameter::{
|
||||
BoolParam, Categorical, CategoricalParam, EnumParam, FloatParam, IntParam, ParamId, Parameter,
|
||||
};
|
||||
pub use pruner::{
|
||||
HyperbandPruner, MedianPruner, NopPruner, PatientPruner, PercentilePruner, Pruner,
|
||||
SuccessiveHalvingPruner, ThresholdPruner, WilcoxonPruner,
|
||||
};
|
||||
pub use sampler::CompletedTrial;
|
||||
pub use sampler::bohb::BohbSampler;
|
||||
#[cfg(feature = "cma-es")]
|
||||
pub use sampler::cma_es::CmaEsSampler;
|
||||
pub use sampler::grid::GridSearchSampler;
|
||||
pub use sampler::motpe::MotpeSampler;
|
||||
pub use sampler::nsga2::Nsga2Sampler;
|
||||
pub use sampler::random::RandomSampler;
|
||||
#[cfg(feature = "sobol")]
|
||||
pub use sampler::sobol::SobolSampler;
|
||||
pub use sampler::tpe::TpeSampler;
|
||||
pub use error::{Result, TpeError};
|
||||
pub use sampler::{CompletedTrial, RandomSampler, Sampler, TpeSampler, TpeSamplerBuilder};
|
||||
pub use study::Study;
|
||||
#[cfg(feature = "serde")]
|
||||
pub use study::StudySnapshot;
|
||||
pub use trial::{AttrValue, Trial};
|
||||
pub use trial::Trial;
|
||||
pub use types::{Direction, TrialState};
|
||||
#[cfg(feature = "visualization")]
|
||||
pub use visualization::generate_html_report;
|
||||
|
||||
/// Convenient wildcard import for the most common types.
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::prelude::*;
|
||||
/// ```
|
||||
pub mod prelude {
|
||||
#[cfg(feature = "derive")]
|
||||
pub use optimizer_derive::Categorical as DeriveCategory;
|
||||
|
||||
pub use crate::error::{Error, Result, TrialPruned};
|
||||
#[cfg(feature = "fanova")]
|
||||
pub use crate::fanova::{FanovaConfig, FanovaResult};
|
||||
pub use crate::multi_objective::{MultiObjectiveStudy, MultiObjectiveTrial};
|
||||
pub use crate::param::ParamValue;
|
||||
pub use crate::parameter::{
|
||||
BoolParam, Categorical, CategoricalParam, EnumParam, FloatParam, IntParam, Parameter,
|
||||
};
|
||||
pub use crate::pruner::{
|
||||
HyperbandPruner, MedianPruner, NopPruner, PatientPruner, PercentilePruner, Pruner,
|
||||
SuccessiveHalvingPruner, ThresholdPruner,
|
||||
};
|
||||
pub use crate::sampler::CompletedTrial;
|
||||
pub use crate::sampler::bohb::BohbSampler;
|
||||
#[cfg(feature = "cma-es")]
|
||||
pub use crate::sampler::cma_es::CmaEsSampler;
|
||||
pub use crate::sampler::grid::GridSearchSampler;
|
||||
pub use crate::sampler::motpe::MotpeSampler;
|
||||
pub use crate::sampler::nsga2::Nsga2Sampler;
|
||||
pub use crate::sampler::random::RandomSampler;
|
||||
#[cfg(feature = "sobol")]
|
||||
pub use crate::sampler::sobol::SobolSampler;
|
||||
pub use crate::sampler::tpe::TpeSampler;
|
||||
pub use crate::study::Study;
|
||||
#[cfg(feature = "serde")]
|
||||
pub use crate::study::StudySnapshot;
|
||||
pub use crate::trial::{AttrValue, Trial};
|
||||
pub use crate::types::Direction;
|
||||
#[cfg(feature = "visualization")]
|
||||
pub use crate::visualization::generate_html_report;
|
||||
}
|
||||
|
||||
@@ -1,412 +0,0 @@
|
||||
//! Multi-objective optimization via a dedicated study type.
|
||||
//!
|
||||
//! [`MultiObjectiveStudy`] manages trials that return multiple objective
|
||||
//! values. It supports arbitrary numbers of objectives with per-objective
|
||||
//! directions (minimize or maximize). Use [`pareto_front()`](MultiObjectiveStudy::pareto_front)
|
||||
//! to retrieve the Pareto-optimal solutions.
|
||||
//!
|
||||
//! # Examples
|
||||
//!
|
||||
//! ```
|
||||
//! use optimizer::Direction;
|
||||
//! use optimizer::multi_objective::MultiObjectiveStudy;
|
||||
//! use optimizer::parameter::{FloatParam, Parameter};
|
||||
//!
|
||||
//! let study = MultiObjectiveStudy::new(vec![Direction::Minimize, Direction::Minimize]);
|
||||
//! let x = FloatParam::new(0.0, 1.0);
|
||||
//!
|
||||
//! study
|
||||
//! .optimize(20, |trial| {
|
||||
//! let xv = x.suggest(trial)?;
|
||||
//! Ok::<_, optimizer::Error>(vec![xv, 1.0 - xv])
|
||||
//! })
|
||||
//! .unwrap();
|
||||
//!
|
||||
//! let front = study.pareto_front();
|
||||
//! assert!(!front.is_empty());
|
||||
//! ```
|
||||
|
||||
use core::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use parking_lot::RwLock;
|
||||
|
||||
use crate::distribution::Distribution;
|
||||
use crate::param::ParamValue;
|
||||
use crate::parameter::{ParamId, Parameter};
|
||||
use crate::pruner::NopPruner;
|
||||
use crate::sampler::random::RandomSampler;
|
||||
use crate::sampler::{CompletedTrial, Sampler};
|
||||
use crate::trial::{AttrValue, Trial};
|
||||
use crate::types::{Direction, TrialState};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MultiObjectiveTrial
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A completed trial with multiple objective values.
|
||||
#[derive(Clone, Debug)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
pub struct MultiObjectiveTrial {
|
||||
/// The unique identifier for this trial.
|
||||
pub id: u64,
|
||||
/// The sampled parameter values, keyed by parameter id.
|
||||
pub params: HashMap<ParamId, ParamValue>,
|
||||
/// The parameter distributions used, keyed by parameter id.
|
||||
pub distributions: HashMap<ParamId, Distribution>,
|
||||
/// Human-readable labels for parameters, keyed by parameter id.
|
||||
pub param_labels: HashMap<ParamId, String>,
|
||||
/// The objective values (one per objective).
|
||||
pub values: Vec<f64>,
|
||||
/// The state of the trial.
|
||||
pub state: TrialState,
|
||||
/// User-defined attributes stored during the trial.
|
||||
pub user_attrs: HashMap<String, AttrValue>,
|
||||
/// Constraint values for this trial (<=0.0 means feasible).
|
||||
#[cfg_attr(feature = "serde", serde(default))]
|
||||
pub constraints: Vec<f64>,
|
||||
}
|
||||
|
||||
impl MultiObjectiveTrial {
|
||||
/// Returns the typed value for the given parameter.
|
||||
///
|
||||
/// Returns `None` if the parameter was not used in this trial.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if the stored value is incompatible with the parameter type.
|
||||
pub fn get<P: Parameter>(&self, param: &P) -> Option<P::Value> {
|
||||
self.params.get(¶m.id()).map(|v| {
|
||||
param
|
||||
.cast_param_value(v)
|
||||
.expect("parameter type mismatch: stored value incompatible with parameter")
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns `true` if all constraints are satisfied (values <= 0.0).
|
||||
///
|
||||
/// A trial with no constraints is considered feasible.
|
||||
#[must_use]
|
||||
pub fn is_feasible(&self) -> bool {
|
||||
self.constraints.iter().all(|&c| c <= 0.0)
|
||||
}
|
||||
|
||||
/// Gets a user attribute by key.
|
||||
#[must_use]
|
||||
pub fn user_attr(&self, key: &str) -> Option<&AttrValue> {
|
||||
self.user_attrs.get(key)
|
||||
}
|
||||
|
||||
/// Returns all user attributes.
|
||||
#[must_use]
|
||||
pub fn user_attrs(&self) -> &HashMap<String, AttrValue> {
|
||||
&self.user_attrs
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MultiObjectiveSampler trait
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Trait for samplers aware of multi-objective history.
|
||||
///
|
||||
/// Separate from [`Sampler`] because NSGA-II needs access to
|
||||
/// `&[MultiObjectiveTrial]` (with vector-valued objectives) and
|
||||
/// `&[Direction]` (one direction per objective).
|
||||
pub trait MultiObjectiveSampler: Send + Sync {
|
||||
/// Samples a parameter value from the given distribution.
|
||||
fn sample(
|
||||
&self,
|
||||
distribution: &Distribution,
|
||||
trial_id: u64,
|
||||
history: &[MultiObjectiveTrial],
|
||||
directions: &[Direction],
|
||||
) -> ParamValue;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// RandomMultiObjectiveSampler
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Default MO sampler that delegates to [`RandomSampler`].
|
||||
pub(crate) struct RandomMultiObjectiveSampler(RandomSampler);
|
||||
|
||||
impl RandomMultiObjectiveSampler {
|
||||
pub(crate) fn new() -> Self {
|
||||
Self(RandomSampler::new())
|
||||
}
|
||||
}
|
||||
|
||||
impl MultiObjectiveSampler for RandomMultiObjectiveSampler {
|
||||
fn sample(
|
||||
&self,
|
||||
distribution: &Distribution,
|
||||
trial_id: u64,
|
||||
_history: &[MultiObjectiveTrial],
|
||||
_directions: &[Direction],
|
||||
) -> ParamValue {
|
||||
self.0.sample(distribution, trial_id, &[])
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MoSamplerBridge — bridges MultiObjectiveSampler to Sampler trait
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Bridges a [`MultiObjectiveSampler`] to the [`Sampler`] trait so that
|
||||
/// `Trial::with_sampler()` can use it.
|
||||
struct MoSamplerBridge {
|
||||
inner: Arc<dyn MultiObjectiveSampler>,
|
||||
history: Arc<RwLock<Vec<MultiObjectiveTrial>>>,
|
||||
directions: Vec<Direction>,
|
||||
}
|
||||
|
||||
impl Sampler for MoSamplerBridge {
|
||||
fn sample(
|
||||
&self,
|
||||
distribution: &Distribution,
|
||||
trial_id: u64,
|
||||
_history: &[CompletedTrial],
|
||||
) -> ParamValue {
|
||||
let mo_history = self.history.read();
|
||||
self.inner
|
||||
.sample(distribution, trial_id, &mo_history, &self.directions)
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MultiObjectiveStudy
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// A study for multi-objective optimization.
|
||||
///
|
||||
/// Manages trials that return multiple objective values. Supports
|
||||
/// arbitrary numbers of objectives with independent minimize/maximize
|
||||
/// directions.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::Direction;
|
||||
/// use optimizer::multi_objective::MultiObjectiveStudy;
|
||||
/// use optimizer::parameter::{FloatParam, Parameter};
|
||||
///
|
||||
/// // Bi-objective: minimize both
|
||||
/// let study = MultiObjectiveStudy::new(vec![Direction::Minimize, Direction::Minimize]);
|
||||
/// let x = FloatParam::new(0.0, 1.0);
|
||||
///
|
||||
/// study
|
||||
/// .optimize(30, |trial| {
|
||||
/// let xv = x.suggest(trial)?;
|
||||
/// Ok::<_, optimizer::Error>(vec![xv, 1.0 - xv])
|
||||
/// })
|
||||
/// .unwrap();
|
||||
///
|
||||
/// let front = study.pareto_front();
|
||||
/// assert!(!front.is_empty());
|
||||
/// ```
|
||||
pub struct MultiObjectiveStudy {
|
||||
directions: Vec<Direction>,
|
||||
sampler: Arc<dyn MultiObjectiveSampler>,
|
||||
completed_trials: Arc<RwLock<Vec<MultiObjectiveTrial>>>,
|
||||
next_trial_id: AtomicU64,
|
||||
}
|
||||
|
||||
impl MultiObjectiveStudy {
|
||||
/// Creates a new multi-objective study with the given directions.
|
||||
///
|
||||
/// Uses a random sampler by default.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `directions` - One direction per objective (minimize or maximize).
|
||||
#[must_use]
|
||||
pub fn new(directions: Vec<Direction>) -> Self {
|
||||
Self {
|
||||
directions,
|
||||
sampler: Arc::new(RandomMultiObjectiveSampler::new()),
|
||||
completed_trials: Arc::new(RwLock::new(Vec::new())),
|
||||
next_trial_id: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a new study with a custom multi-objective sampler.
|
||||
#[must_use]
|
||||
pub fn with_sampler(
|
||||
directions: Vec<Direction>,
|
||||
sampler: impl MultiObjectiveSampler + 'static,
|
||||
) -> Self {
|
||||
Self {
|
||||
directions,
|
||||
sampler: Arc::new(sampler),
|
||||
completed_trials: Arc::new(RwLock::new(Vec::new())),
|
||||
next_trial_id: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the optimization directions.
|
||||
#[must_use]
|
||||
pub fn directions(&self) -> &[Direction] {
|
||||
&self.directions
|
||||
}
|
||||
|
||||
/// Returns the number of objectives.
|
||||
#[must_use]
|
||||
pub fn n_objectives(&self) -> usize {
|
||||
self.directions.len()
|
||||
}
|
||||
|
||||
/// Returns the number of completed trials.
|
||||
#[must_use]
|
||||
pub fn n_trials(&self) -> usize {
|
||||
self.completed_trials.read().len()
|
||||
}
|
||||
|
||||
/// Returns all completed trials.
|
||||
#[must_use]
|
||||
pub fn trials(&self) -> Vec<MultiObjectiveTrial> {
|
||||
self.completed_trials.read().clone()
|
||||
}
|
||||
|
||||
/// Returns the Pareto-optimal trials (front 0).
|
||||
#[must_use]
|
||||
pub fn pareto_front(&self) -> Vec<MultiObjectiveTrial> {
|
||||
let trials = self.completed_trials.read();
|
||||
let complete: Vec<_> = trials
|
||||
.iter()
|
||||
.filter(|t| t.state == TrialState::Complete)
|
||||
.collect();
|
||||
|
||||
if complete.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let values: Vec<Vec<f64>> = complete.iter().map(|t| t.values.clone()).collect();
|
||||
let fronts = crate::pareto::fast_non_dominated_sort(&values, &self.directions);
|
||||
|
||||
if fronts.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
fronts[0].iter().map(|&i| complete[i].clone()).collect()
|
||||
}
|
||||
|
||||
/// Creates a new trial wired to the study's MO sampler.
|
||||
fn create_trial(&self) -> Trial {
|
||||
let id = self.next_trial_id.fetch_add(1, Ordering::SeqCst);
|
||||
|
||||
let bridge: Arc<dyn Sampler> = Arc::new(MoSamplerBridge {
|
||||
inner: Arc::clone(&self.sampler),
|
||||
history: Arc::clone(&self.completed_trials),
|
||||
directions: self.directions.clone(),
|
||||
});
|
||||
|
||||
// Dummy f64 history — the bridge ignores it.
|
||||
let dummy_history: Arc<RwLock<Vec<CompletedTrial<f64>>>> =
|
||||
Arc::new(RwLock::new(Vec::new()));
|
||||
|
||||
Trial::with_sampler(id, bridge, dummy_history, Arc::new(NopPruner))
|
||||
}
|
||||
|
||||
/// Records a completed trial.
|
||||
fn complete_trial(&self, mut trial: Trial, values: Vec<f64>) {
|
||||
trial.set_complete();
|
||||
let mo_trial = MultiObjectiveTrial {
|
||||
id: trial.id(),
|
||||
params: trial.params().clone(),
|
||||
distributions: trial.distributions().clone(),
|
||||
param_labels: trial.param_labels().clone(),
|
||||
values,
|
||||
state: TrialState::Complete,
|
||||
user_attrs: trial.user_attrs().clone(),
|
||||
constraints: trial.constraint_values().to_vec(),
|
||||
};
|
||||
self.completed_trials.write().push(mo_trial);
|
||||
}
|
||||
|
||||
/// Records a failed trial (not stored in history).
|
||||
fn fail_trial(trial: &mut Trial) {
|
||||
trial.set_failed();
|
||||
}
|
||||
|
||||
/// Request a new trial for the ask/tell interface.
|
||||
///
|
||||
/// After creating the trial, suggest parameters on it, evaluate your
|
||||
/// objective externally, then pass the trial back to [`tell()`](Self::tell).
|
||||
pub fn ask(&self) -> Trial {
|
||||
self.create_trial()
|
||||
}
|
||||
|
||||
/// Report the result of a trial obtained from [`ask()`](Self::ask).
|
||||
///
|
||||
/// Pass `Ok(values)` for a successful evaluation or `Err(reason)` for a failure.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns `ObjectiveDimensionMismatch` if the number of values doesn't
|
||||
/// match the number of directions.
|
||||
pub fn tell(
|
||||
&self,
|
||||
mut trial: Trial,
|
||||
result: core::result::Result<Vec<f64>, impl ToString>,
|
||||
) -> crate::Result<()> {
|
||||
if let Ok(values) = result {
|
||||
if values.len() != self.directions.len() {
|
||||
return Err(crate::Error::ObjectiveDimensionMismatch {
|
||||
expected: self.directions.len(),
|
||||
got: values.len(),
|
||||
});
|
||||
}
|
||||
self.complete_trial(trial, values);
|
||||
} else {
|
||||
Self::fail_trial(&mut trial);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Runs multi-objective optimization for `n_trials` trials.
|
||||
///
|
||||
/// The objective function must return a `Vec<f64>` with one value per
|
||||
/// objective.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns `ObjectiveDimensionMismatch` if the objective returns the wrong
|
||||
/// number of values. Returns `NoCompletedTrials` if all trials fail.
|
||||
pub fn optimize<F, E>(&self, n_trials: usize, mut objective: F) -> crate::Result<()>
|
||||
where
|
||||
F: FnMut(&mut Trial) -> core::result::Result<Vec<f64>, E>,
|
||||
E: ToString,
|
||||
{
|
||||
for _ in 0..n_trials {
|
||||
let mut trial = self.create_trial();
|
||||
|
||||
match objective(&mut trial) {
|
||||
Ok(values) => {
|
||||
if values.len() != self.directions.len() {
|
||||
return Err(crate::Error::ObjectiveDimensionMismatch {
|
||||
expected: self.directions.len(),
|
||||
got: values.len(),
|
||||
});
|
||||
}
|
||||
self.complete_trial(trial, values);
|
||||
}
|
||||
Err(_) => {
|
||||
Self::fail_trial(&mut trial);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let has_complete = self
|
||||
.completed_trials
|
||||
.read()
|
||||
.iter()
|
||||
.any(|t| t.state == TrialState::Complete);
|
||||
if !has_complete {
|
||||
return Err(crate::Error::NoCompletedTrials);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,6 @@
|
||||
/// For categorical parameters, the `Categorical` variant stores
|
||||
/// the index into the choices array.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
pub enum ParamValue {
|
||||
/// A floating-point parameter value.
|
||||
Float(f64),
|
||||
@@ -15,13 +14,3 @@ pub enum ParamValue {
|
||||
/// A categorical parameter value, stored as an index into the choices array.
|
||||
Categorical(usize),
|
||||
}
|
||||
|
||||
impl core::fmt::Display for ParamValue {
|
||||
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
|
||||
match self {
|
||||
Self::Float(v) => write!(f, "{v}"),
|
||||
Self::Int(v) => write!(f, "{v}"),
|
||||
Self::Categorical(v) => write!(f, "category({v})"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
-1039
File diff suppressed because it is too large
Load Diff
-593
@@ -1,593 +0,0 @@
|
||||
//! Pareto front analysis utilities for multi-objective optimization.
|
||||
//!
|
||||
//! Provides functions for analyzing and working with Pareto fronts:
|
||||
//!
|
||||
//! - [`hypervolume`] — measure the quality of a Pareto front
|
||||
//! - [`non_dominated_sort`] — rank solutions into successive fronts
|
||||
//! - [`pareto_front_indices`] — filter to non-dominated solutions only
|
||||
//! - [`crowding_distance`] — measure diversity within a front
|
||||
//!
|
||||
//! Internally also provides fast non-dominated sorting (Deb et al., 2002)
|
||||
//! used by [`MultiObjectiveStudy::pareto_front()`](crate::MultiObjectiveStudy::pareto_front)
|
||||
//! and [`Nsga2Sampler`](crate::Nsga2Sampler).
|
||||
|
||||
use crate::types::Direction;
|
||||
|
||||
/// Returns `true` if solution `a` Pareto-dominates solution `b`.
|
||||
///
|
||||
/// A solution dominates another if it is at least as good in all objectives
|
||||
/// and strictly better in at least one, respecting the given directions.
|
||||
#[allow(clippy::module_name_repetitions)]
|
||||
pub(crate) fn dominates(a: &[f64], b: &[f64], directions: &[Direction]) -> bool {
|
||||
debug_assert_eq!(a.len(), b.len());
|
||||
debug_assert_eq!(a.len(), directions.len());
|
||||
|
||||
let mut strictly_better = false;
|
||||
for ((&av, &bv), dir) in a.iter().zip(b.iter()).zip(directions.iter()) {
|
||||
let better = match dir {
|
||||
Direction::Minimize => av < bv,
|
||||
Direction::Maximize => av > bv,
|
||||
};
|
||||
let worse = match dir {
|
||||
Direction::Minimize => av > bv,
|
||||
Direction::Maximize => av < bv,
|
||||
};
|
||||
if worse {
|
||||
return false;
|
||||
}
|
||||
if better {
|
||||
strictly_better = true;
|
||||
}
|
||||
}
|
||||
strictly_better
|
||||
}
|
||||
|
||||
/// Constrained dominance: feasible beats infeasible, among infeasible
|
||||
/// prefer lower total constraint violation, among feasible use Pareto dominance.
|
||||
pub(crate) fn constrained_dominates(
|
||||
a_values: &[f64],
|
||||
b_values: &[f64],
|
||||
a_constraints: &[f64],
|
||||
b_constraints: &[f64],
|
||||
directions: &[Direction],
|
||||
) -> bool {
|
||||
let a_feasible = a_constraints.iter().all(|&c| c <= 0.0);
|
||||
let b_feasible = b_constraints.iter().all(|&c| c <= 0.0);
|
||||
|
||||
match (a_feasible, b_feasible) {
|
||||
(true, false) => true,
|
||||
(false, true) => false,
|
||||
(false, false) => {
|
||||
let a_violation: f64 = a_constraints.iter().map(|c| c.max(0.0)).sum();
|
||||
let b_violation: f64 = b_constraints.iter().map(|c| c.max(0.0)).sum();
|
||||
a_violation < b_violation
|
||||
}
|
||||
(true, true) => dominates(a_values, b_values, directions),
|
||||
}
|
||||
}
|
||||
|
||||
/// Fast non-dominated sorting (Deb et al., 2002).
|
||||
///
|
||||
/// Returns `Vec<Vec<usize>>` where `fronts[0]` is the Pareto front,
|
||||
/// each inner vec contains indices into `values`.
|
||||
///
|
||||
/// Complexity: O(M * N^2) where M = objectives, N = solutions.
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
pub(crate) fn fast_non_dominated_sort(
|
||||
values: &[Vec<f64>],
|
||||
directions: &[Direction],
|
||||
) -> Vec<Vec<usize>> {
|
||||
fast_non_dominated_sort_constrained(values, directions, &[])
|
||||
}
|
||||
|
||||
/// Fast non-dominated sorting with constraint support.
|
||||
///
|
||||
/// `constraints` is either empty (no constraints) or has the same length
|
||||
/// as `values`, where each entry is the constraint vector for that solution.
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
pub(crate) fn fast_non_dominated_sort_constrained(
|
||||
values: &[Vec<f64>],
|
||||
directions: &[Direction],
|
||||
constraints: &[Vec<f64>],
|
||||
) -> Vec<Vec<usize>> {
|
||||
let n = values.len();
|
||||
if n == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let has_constraints = !constraints.is_empty();
|
||||
let empty_constraints: Vec<f64> = Vec::new();
|
||||
|
||||
// S_p: set of solutions dominated by p
|
||||
let mut dominated_by: Vec<Vec<usize>> = vec![Vec::new(); n];
|
||||
// n_p: domination count for p
|
||||
let mut domination_count: Vec<usize> = vec![0; n];
|
||||
|
||||
for i in 0..n {
|
||||
for j in (i + 1)..n {
|
||||
let (a_c, b_c) = if has_constraints {
|
||||
(&constraints[i], &constraints[j])
|
||||
} else {
|
||||
(&empty_constraints, &empty_constraints)
|
||||
};
|
||||
|
||||
let i_dom_j = if has_constraints {
|
||||
constrained_dominates(&values[i], &values[j], a_c, b_c, directions)
|
||||
} else {
|
||||
dominates(&values[i], &values[j], directions)
|
||||
};
|
||||
let j_dom_i = if has_constraints {
|
||||
constrained_dominates(&values[j], &values[i], b_c, a_c, directions)
|
||||
} else {
|
||||
dominates(&values[j], &values[i], directions)
|
||||
};
|
||||
|
||||
if i_dom_j {
|
||||
dominated_by[i].push(j);
|
||||
domination_count[j] += 1;
|
||||
} else if j_dom_i {
|
||||
dominated_by[j].push(i);
|
||||
domination_count[i] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut fronts: Vec<Vec<usize>> = Vec::new();
|
||||
let mut current_front: Vec<usize> = (0..n).filter(|&i| domination_count[i] == 0).collect();
|
||||
|
||||
while !current_front.is_empty() {
|
||||
let mut next_front: Vec<usize> = Vec::new();
|
||||
for &p in ¤t_front {
|
||||
for &q in &dominated_by[p] {
|
||||
domination_count[q] -= 1;
|
||||
if domination_count[q] == 0 {
|
||||
next_front.push(q);
|
||||
}
|
||||
}
|
||||
}
|
||||
fronts.push(current_front);
|
||||
current_front = next_front;
|
||||
}
|
||||
|
||||
fronts
|
||||
}
|
||||
|
||||
/// Crowding distance for one front (index-based, internal API).
|
||||
///
|
||||
/// Boundary solutions get `f64::INFINITY`. Returns one distance value per
|
||||
/// solution in the front, in the same order as `front_indices`.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
pub(crate) fn crowding_distance_indexed(front_indices: &[usize], values: &[Vec<f64>]) -> Vec<f64> {
|
||||
let n = front_indices.len();
|
||||
if n <= 2 {
|
||||
return vec![f64::INFINITY; n];
|
||||
}
|
||||
|
||||
let m = values[front_indices[0]].len(); // number of objectives
|
||||
let mut distances = vec![0.0_f64; n];
|
||||
|
||||
// Helper to look up objective value for a front member.
|
||||
let val = |front_pos: usize, obj: usize| -> f64 { values[front_indices[front_pos]][obj] };
|
||||
|
||||
for obj in 0..m {
|
||||
// Sort front positions by this objective
|
||||
let mut sorted: Vec<usize> = (0..n).collect();
|
||||
sorted.sort_by(|&a, &b| {
|
||||
val(a, obj)
|
||||
.partial_cmp(&val(b, obj))
|
||||
.unwrap_or(core::cmp::Ordering::Equal)
|
||||
});
|
||||
|
||||
// Boundary solutions get infinity
|
||||
distances[sorted[0]] = f64::INFINITY;
|
||||
distances[sorted[n - 1]] = f64::INFINITY;
|
||||
|
||||
let range = val(sorted[n - 1], obj) - val(sorted[0], obj);
|
||||
if range > 0.0 {
|
||||
for i in 1..(n - 1) {
|
||||
distances[sorted[i]] += (val(sorted[i + 1], obj) - val(sorted[i - 1], obj)) / range;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
distances
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Public API
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Compute the hypervolume indicator of a Pareto front.
|
||||
///
|
||||
/// The hypervolume is the volume of the objective space dominated by
|
||||
/// the Pareto front and bounded by a reference point. Higher values
|
||||
/// indicate a better front.
|
||||
///
|
||||
/// Each entry in `front` is one solution's objective values.
|
||||
/// `reference_point` should be worse than all front members in every
|
||||
/// objective (e.g., the worst acceptable values).
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics (in debug) if dimensions of `front`, `reference_point`, and
|
||||
/// `directions` are inconsistent.
|
||||
#[must_use]
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
pub fn hypervolume(front: &[Vec<f64>], reference_point: &[f64], directions: &[Direction]) -> f64 {
|
||||
if front.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
let d = reference_point.len();
|
||||
debug_assert!(front.iter().all(|p| p.len() == d));
|
||||
debug_assert_eq!(d, directions.len());
|
||||
|
||||
// Normalize to minimize-space (negate maximized objectives).
|
||||
let normalized: Vec<Vec<f64>> = front
|
||||
.iter()
|
||||
.map(|p| {
|
||||
p.iter()
|
||||
.zip(directions)
|
||||
.map(|(&v, dir)| match dir {
|
||||
Direction::Minimize => v,
|
||||
Direction::Maximize => -v,
|
||||
})
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
|
||||
let ref_norm: Vec<f64> = reference_point
|
||||
.iter()
|
||||
.zip(directions)
|
||||
.map(|(&v, dir)| match dir {
|
||||
Direction::Minimize => v,
|
||||
Direction::Maximize => -v,
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Keep only points strictly dominated by the reference point.
|
||||
let filtered: Vec<Vec<f64>> = normalized
|
||||
.into_iter()
|
||||
.filter(|p| p.iter().zip(&ref_norm).all(|(&pv, &rv)| pv < rv))
|
||||
.collect();
|
||||
|
||||
if filtered.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
hv_recursive(&filtered, &ref_norm)
|
||||
}
|
||||
|
||||
/// Recursive hypervolume via slicing on the last objective.
|
||||
///
|
||||
/// All points are in minimize-space and dominated by `reference`.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
fn hv_recursive(points: &[Vec<f64>], reference: &[f64]) -> f64 {
|
||||
let d = reference.len();
|
||||
|
||||
// Base case: 1-D hypervolume is just the gap from the best point to ref.
|
||||
if d == 1 {
|
||||
let min_val = points.iter().map(|p| p[0]).fold(f64::INFINITY, f64::min);
|
||||
return (reference[0] - min_val).max(0.0);
|
||||
}
|
||||
|
||||
// Single point: hypervolume is the product of gaps.
|
||||
if points.len() == 1 {
|
||||
return points[0]
|
||||
.iter()
|
||||
.zip(reference)
|
||||
.map(|(&p, &r)| (r - p).max(0.0))
|
||||
.product();
|
||||
}
|
||||
|
||||
// Sort by last objective ascending.
|
||||
let mut sorted: Vec<&Vec<f64>> = points.iter().collect();
|
||||
sorted.sort_by(|a, b| {
|
||||
a[d - 1]
|
||||
.partial_cmp(&b[d - 1])
|
||||
.unwrap_or(core::cmp::Ordering::Equal)
|
||||
});
|
||||
|
||||
let sub_ref: Vec<f64> = reference[..d - 1].to_vec();
|
||||
let mut result = 0.0;
|
||||
|
||||
for i in 0..sorted.len() {
|
||||
let height = if i + 1 < sorted.len() {
|
||||
sorted[i + 1][d - 1] - sorted[i][d - 1]
|
||||
} else {
|
||||
reference[d - 1] - sorted[i][d - 1]
|
||||
};
|
||||
|
||||
if height <= 0.0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Project points[0..=i] onto the first d-1 dimensions and
|
||||
// keep only the non-dominated subset.
|
||||
let projected: Vec<Vec<f64>> = sorted[..=i].iter().map(|p| p[..d - 1].to_vec()).collect();
|
||||
let non_dom = non_dominated_minimize(&projected);
|
||||
|
||||
if !non_dom.is_empty() {
|
||||
result += height * hv_recursive(&non_dom, &sub_ref);
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
/// Return the non-dominated subset of `points` in minimize-space.
|
||||
fn non_dominated_minimize(points: &[Vec<f64>]) -> Vec<Vec<f64>> {
|
||||
let mut result = Vec::new();
|
||||
'outer: for (i, p) in points.iter().enumerate() {
|
||||
for (j, q) in points.iter().enumerate() {
|
||||
if i == j {
|
||||
continue;
|
||||
}
|
||||
// Check if q dominates p (all <=, at least one <).
|
||||
let mut all_leq = true;
|
||||
let mut any_lt = false;
|
||||
for (&qv, &pv) in q.iter().zip(p.iter()) {
|
||||
if qv > pv {
|
||||
all_leq = false;
|
||||
break;
|
||||
}
|
||||
if qv < pv {
|
||||
any_lt = true;
|
||||
}
|
||||
}
|
||||
if all_leq && any_lt {
|
||||
continue 'outer;
|
||||
}
|
||||
}
|
||||
result.push(p.clone());
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Compute non-dominated sorting of a set of solutions.
|
||||
///
|
||||
/// Returns a vec of fronts, where `fronts[0]` is the Pareto front,
|
||||
/// `fronts[1]` is the next best, etc. Each inner vec contains indices
|
||||
/// into the original `solutions` slice.
|
||||
///
|
||||
/// Uses the fast non-dominated sorting algorithm from
|
||||
/// Deb et al. (2002) with O(M N²) complexity.
|
||||
#[must_use]
|
||||
pub fn non_dominated_sort(solutions: &[Vec<f64>], directions: &[Direction]) -> Vec<Vec<usize>> {
|
||||
fast_non_dominated_sort(solutions, directions)
|
||||
}
|
||||
|
||||
/// Filter solutions to return only non-dominated (Pareto-optimal) indices.
|
||||
///
|
||||
/// Equivalent to `non_dominated_sort(solutions, directions)[0]` but
|
||||
/// communicates the intent more clearly.
|
||||
#[must_use]
|
||||
pub fn pareto_front_indices(solutions: &[Vec<f64>], directions: &[Direction]) -> Vec<usize> {
|
||||
let fronts = fast_non_dominated_sort(solutions, directions);
|
||||
fronts.into_iter().next().unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Compute crowding distance for diversity measurement.
|
||||
///
|
||||
/// Returns one distance value per solution in `front` (same order).
|
||||
/// Boundary solutions (best/worst in any objective) receive
|
||||
/// [`f64::INFINITY`]. Interior solutions get a finite positive value
|
||||
/// proportional to the gap between their neighbors.
|
||||
///
|
||||
/// `directions` is accepted for API consistency but does not affect
|
||||
/// the result, since crowding distance measures spacing regardless of
|
||||
/// optimization direction.
|
||||
#[must_use]
|
||||
#[allow(clippy::cast_precision_loss, clippy::needless_range_loop)]
|
||||
pub fn crowding_distance(front: &[Vec<f64>], _directions: &[Direction]) -> Vec<f64> {
|
||||
let n = front.len();
|
||||
if n <= 2 {
|
||||
return vec![f64::INFINITY; n];
|
||||
}
|
||||
|
||||
let m = front[0].len();
|
||||
let mut distances = vec![0.0_f64; n];
|
||||
|
||||
for obj in 0..m {
|
||||
let mut sorted: Vec<usize> = (0..n).collect();
|
||||
sorted.sort_by(|&a, &b| {
|
||||
front[a][obj]
|
||||
.partial_cmp(&front[b][obj])
|
||||
.unwrap_or(core::cmp::Ordering::Equal)
|
||||
});
|
||||
|
||||
distances[sorted[0]] = f64::INFINITY;
|
||||
distances[sorted[n - 1]] = f64::INFINITY;
|
||||
|
||||
let range = front[sorted[n - 1]][obj] - front[sorted[0]][obj];
|
||||
if range > 0.0 {
|
||||
for i in 1..(n - 1) {
|
||||
distances[sorted[i]] +=
|
||||
(front[sorted[i + 1]][obj] - front[sorted[i - 1]][obj]) / range;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
distances
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_dominates_basic() {
|
||||
let dirs = [Direction::Minimize, Direction::Minimize];
|
||||
assert!(dominates(&[1.0, 1.0], &[2.0, 2.0], &dirs));
|
||||
assert!(!dominates(&[2.0, 2.0], &[1.0, 1.0], &dirs));
|
||||
// Equal does not dominate
|
||||
assert!(!dominates(&[1.0, 1.0], &[1.0, 1.0], &dirs));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dominates_incomparable() {
|
||||
let dirs = [Direction::Minimize, Direction::Minimize];
|
||||
assert!(!dominates(&[1.0, 3.0], &[3.0, 1.0], &dirs));
|
||||
assert!(!dominates(&[3.0, 1.0], &[1.0, 3.0], &dirs));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dominates_maximize() {
|
||||
let dirs = [Direction::Maximize, Direction::Minimize];
|
||||
// a = (5, 1) vs b = (3, 2): a is better in both
|
||||
assert!(dominates(&[5.0, 1.0], &[3.0, 2.0], &dirs));
|
||||
assert!(!dominates(&[3.0, 2.0], &[5.0, 1.0], &dirs));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_nds_known() {
|
||||
let values = vec![
|
||||
vec![1.0, 5.0], // front 0
|
||||
vec![5.0, 1.0], // front 0
|
||||
vec![3.0, 3.0], // front 0 (non-dominated)
|
||||
vec![4.0, 4.0], // front 1 (dominated by #2)
|
||||
vec![6.0, 6.0], // front 2
|
||||
];
|
||||
let dirs = [Direction::Minimize, Direction::Minimize];
|
||||
let fronts = fast_non_dominated_sort(&values, &dirs);
|
||||
|
||||
assert_eq!(fronts.len(), 3);
|
||||
let mut f0 = fronts[0].clone();
|
||||
f0.sort_unstable();
|
||||
assert_eq!(f0, vec![0, 1, 2]);
|
||||
assert_eq!(fronts[1], vec![3]);
|
||||
assert_eq!(fronts[2], vec![4]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crowding_indexed_boundaries() {
|
||||
let values = vec![vec![1.0, 5.0], vec![3.0, 3.0], vec![5.0, 1.0]];
|
||||
let front = vec![0, 1, 2];
|
||||
let cd = crowding_distance_indexed(&front, &values);
|
||||
assert!(cd[0].is_infinite());
|
||||
assert!(cd[2].is_infinite());
|
||||
assert!(cd[1].is_finite());
|
||||
assert!(cd[1] > 0.0);
|
||||
}
|
||||
|
||||
// ---- Public API tests ----
|
||||
|
||||
#[test]
|
||||
fn test_hypervolume_2d_minimize() {
|
||||
// Front: (1,3), (2,2), (3,1) with ref (4,4) — all minimize
|
||||
let front = vec![vec![1.0, 3.0], vec![2.0, 2.0], vec![3.0, 1.0]];
|
||||
let dirs = [Direction::Minimize, Direction::Minimize];
|
||||
let hv = hypervolume(&front, &[4.0, 4.0], &dirs);
|
||||
// Strip 1: x=[1,2), h=4-3=1 → area=1
|
||||
// Strip 2: x=[2,3), h=4-2=2 → area=2
|
||||
// Strip 3: x=[3,4], h=4-1=3 → area=3
|
||||
// Total = 6
|
||||
assert!((hv - 6.0).abs() < 1e-10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hypervolume_2d_maximize() {
|
||||
// Front: (3,1), (2,2), (1,3) with ref (0,0) — all maximize
|
||||
let front = vec![vec![3.0, 1.0], vec![2.0, 2.0], vec![1.0, 3.0]];
|
||||
let dirs = [Direction::Maximize, Direction::Maximize];
|
||||
let hv = hypervolume(&front, &[0.0, 0.0], &dirs);
|
||||
// In negate-space: points become (-3,-1),(-2,-2),(-1,-3), ref=(0,0)
|
||||
// Same geometry as minimize test above → area = 6
|
||||
assert!((hv - 6.0).abs() < 1e-10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hypervolume_single_point() {
|
||||
let front = vec![vec![1.0, 1.0]];
|
||||
let dirs = [Direction::Minimize, Direction::Minimize];
|
||||
let hv = hypervolume(&front, &[3.0, 3.0], &dirs);
|
||||
// Rectangle: (3-1) * (3-1) = 4
|
||||
assert!((hv - 4.0).abs() < 1e-10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hypervolume_empty_front() {
|
||||
let front: Vec<Vec<f64>> = vec![];
|
||||
let dirs = [Direction::Minimize];
|
||||
assert!(hypervolume(&front, &[1.0], &dirs).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hypervolume_point_at_ref() {
|
||||
// Point not strictly better than ref → contributes nothing
|
||||
let front = vec![vec![5.0, 5.0]];
|
||||
let dirs = [Direction::Minimize, Direction::Minimize];
|
||||
let hv = hypervolume(&front, &[5.0, 5.0], &dirs);
|
||||
assert!(hv.abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hypervolume_3d() {
|
||||
// Single point in 3D: (1,1,1) with ref (2,2,2)
|
||||
let front = vec![vec![1.0, 1.0, 1.0]];
|
||||
let dirs = [
|
||||
Direction::Minimize,
|
||||
Direction::Minimize,
|
||||
Direction::Minimize,
|
||||
];
|
||||
let hv = hypervolume(&front, &[2.0, 2.0, 2.0], &dirs);
|
||||
assert!((hv - 1.0).abs() < 1e-10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_non_dominated_sort_public() {
|
||||
let values = vec![
|
||||
vec![1.0, 5.0],
|
||||
vec![5.0, 1.0],
|
||||
vec![3.0, 3.0],
|
||||
vec![4.0, 4.0],
|
||||
];
|
||||
let dirs = [Direction::Minimize, Direction::Minimize];
|
||||
let fronts = non_dominated_sort(&values, &dirs);
|
||||
assert_eq!(fronts.len(), 2);
|
||||
let mut f0 = fronts[0].clone();
|
||||
f0.sort_unstable();
|
||||
assert_eq!(f0, vec![0, 1, 2]);
|
||||
assert_eq!(fronts[1], vec![3]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pareto_front_indices_basic() {
|
||||
let values = vec![
|
||||
vec![1.0, 5.0],
|
||||
vec![5.0, 1.0],
|
||||
vec![3.0, 3.0],
|
||||
vec![4.0, 4.0],
|
||||
];
|
||||
let dirs = [Direction::Minimize, Direction::Minimize];
|
||||
let mut idx = pareto_front_indices(&values, &dirs);
|
||||
idx.sort_unstable();
|
||||
assert_eq!(idx, vec![0, 1, 2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_pareto_front_indices_empty() {
|
||||
let values: Vec<Vec<f64>> = vec![];
|
||||
let dirs = [Direction::Minimize];
|
||||
assert!(pareto_front_indices(&values, &dirs).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crowding_distance_public() {
|
||||
let front = vec![vec![1.0, 5.0], vec![3.0, 3.0], vec![5.0, 1.0]];
|
||||
let dirs = [Direction::Minimize, Direction::Minimize];
|
||||
let cd = crowding_distance(&front, &dirs);
|
||||
assert!(cd[0].is_infinite());
|
||||
assert!(cd[2].is_infinite());
|
||||
assert!(cd[1].is_finite());
|
||||
assert!(cd[1] > 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crowding_distance_single_point() {
|
||||
let front = vec![vec![2.0, 3.0]];
|
||||
let dirs = [Direction::Minimize, Direction::Minimize];
|
||||
let cd = crowding_distance(&front, &dirs);
|
||||
assert_eq!(cd.len(), 1);
|
||||
assert!(cd[0].is_infinite());
|
||||
}
|
||||
}
|
||||
@@ -1,559 +0,0 @@
|
||||
use core::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use super::Pruner;
|
||||
use crate::sampler::CompletedTrial;
|
||||
use crate::types::{Direction, TrialState};
|
||||
|
||||
/// Hyperband pruner that manages multiple Successive Halving brackets.
|
||||
///
|
||||
/// Hyperband addresses SHA's sensitivity to the `min_resource` choice by
|
||||
/// running multiple brackets, each with a different tradeoff between the
|
||||
/// number of configurations and the starting budget:
|
||||
///
|
||||
/// - Bracket 0: many trials, very small starting budget (aggressive pruning)
|
||||
/// - Bracket 1: fewer trials, larger starting budget (moderate pruning)
|
||||
/// - ...
|
||||
/// - Bracket `s_max`: few trials, full budget (no pruning)
|
||||
///
|
||||
/// Trials are assigned to brackets in round-robin fashion. Each bracket
|
||||
/// runs SHA with its own `min_resource` and rung schedule.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::Direction;
|
||||
/// use optimizer::pruner::HyperbandPruner;
|
||||
///
|
||||
/// let pruner = HyperbandPruner::new()
|
||||
/// .min_resource(1)
|
||||
/// .max_resource(81)
|
||||
/// .reduction_factor(3)
|
||||
/// .direction(Direction::Minimize);
|
||||
/// ```
|
||||
pub struct HyperbandPruner {
|
||||
min_resource: u64,
|
||||
max_resource: u64,
|
||||
reduction_factor: u64,
|
||||
direction: Direction,
|
||||
/// Tracks which bracket each trial belongs to.
|
||||
trial_brackets: Mutex<HashMap<u64, usize>>,
|
||||
/// Counter for round-robin bracket assignment.
|
||||
next_bracket: AtomicU64,
|
||||
}
|
||||
|
||||
impl HyperbandPruner {
|
||||
/// Create a new `HyperbandPruner` with default parameters.
|
||||
///
|
||||
/// Defaults: `min_resource=1`, `max_resource=81`, `reduction_factor=3`,
|
||||
/// `direction=Minimize`.
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
min_resource: 1,
|
||||
max_resource: 81,
|
||||
reduction_factor: 3,
|
||||
direction: Direction::Minimize,
|
||||
trial_brackets: Mutex::new(HashMap::new()),
|
||||
next_bracket: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the minimum resource (budget) per trial.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `r` is 0.
|
||||
#[must_use]
|
||||
pub fn min_resource(mut self, r: u64) -> Self {
|
||||
assert!(r > 0, "min_resource must be > 0, got {r}");
|
||||
self.min_resource = r;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the maximum resource (budget) per trial.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `r` is 0.
|
||||
#[must_use]
|
||||
pub fn max_resource(mut self, r: u64) -> Self {
|
||||
assert!(r > 0, "max_resource must be > 0, got {r}");
|
||||
self.max_resource = r;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the reduction factor (eta). At each rung, the top 1/eta trials survive.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `eta` is less than 2.
|
||||
#[must_use]
|
||||
pub fn reduction_factor(mut self, eta: u64) -> Self {
|
||||
assert!(eta >= 2, "reduction_factor must be >= 2, got {eta}");
|
||||
self.reduction_factor = eta;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the optimization direction.
|
||||
#[must_use]
|
||||
pub fn direction(mut self, d: Direction) -> Self {
|
||||
self.direction = d;
|
||||
self
|
||||
}
|
||||
|
||||
/// Compute `s_max = floor(log(max_resource / min_resource) / log(eta))`.
|
||||
#[allow(
|
||||
clippy::cast_precision_loss,
|
||||
clippy::cast_possible_truncation,
|
||||
clippy::cast_sign_loss
|
||||
)]
|
||||
fn s_max(&self) -> u64 {
|
||||
let eta = self.reduction_factor as f64;
|
||||
let ratio = self.max_resource as f64 / self.min_resource as f64;
|
||||
(ratio.ln() / eta.ln()).floor() as u64
|
||||
}
|
||||
|
||||
/// Compute the rung steps for a given bracket `s`.
|
||||
///
|
||||
/// For bracket `s`, the starting resource is `max_resource / eta^(s_max - s)`,
|
||||
/// and rungs are spaced at powers of eta from there up to `max_resource`.
|
||||
#[allow(
|
||||
clippy::cast_precision_loss,
|
||||
clippy::cast_possible_truncation,
|
||||
clippy::cast_sign_loss
|
||||
)]
|
||||
fn rung_steps_for_bracket(&self, bracket: usize) -> Vec<u64> {
|
||||
let s_max = self.s_max();
|
||||
let eta = self.reduction_factor as f64;
|
||||
|
||||
// Starting resource for this bracket
|
||||
let exponent = s_max.saturating_sub(bracket as u64);
|
||||
let min_resource_bracket =
|
||||
(self.max_resource as f64 / eta.powi(exponent as i32)).ceil() as u64;
|
||||
|
||||
let mut steps = Vec::new();
|
||||
let mut rung: u32 = 0;
|
||||
while let Some(power) = self.reduction_factor.checked_pow(rung) {
|
||||
let step = min_resource_bracket.saturating_mul(power);
|
||||
if step > self.max_resource {
|
||||
break;
|
||||
}
|
||||
steps.push(step);
|
||||
rung += 1;
|
||||
}
|
||||
steps
|
||||
}
|
||||
|
||||
/// Assign a trial to a bracket (round-robin) and return the bracket index.
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
fn assign_bracket(&self, trial_id: u64) -> usize {
|
||||
let n_brackets = (self.s_max() + 1) as usize;
|
||||
let mut map = self.trial_brackets.lock().expect("lock poisoned");
|
||||
*map.entry(trial_id).or_insert_with(|| {
|
||||
let idx = self.next_bracket.fetch_add(1, Ordering::Relaxed);
|
||||
(idx as usize) % n_brackets
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for HyperbandPruner {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
impl Pruner for HyperbandPruner {
|
||||
fn should_prune(
|
||||
&self,
|
||||
trial_id: u64,
|
||||
step: u64,
|
||||
intermediate_values: &[(u64, f64)],
|
||||
completed_trials: &[CompletedTrial],
|
||||
) -> bool {
|
||||
let bracket = self.assign_bracket(trial_id);
|
||||
let rungs = self.rung_steps_for_bracket(bracket);
|
||||
|
||||
// Find the highest rung step <= current step
|
||||
let Some(&rung_step) = rungs.iter().rev().find(|&&r| r <= step) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
// Never prune at the last rung (full budget)
|
||||
if rung_step >= self.max_resource {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Get the current trial's value at this rung step
|
||||
let current_value =
|
||||
if let Some(&(_, v)) = intermediate_values.iter().find(|(s, _)| *s == rung_step) {
|
||||
v
|
||||
} else if let Some(&(_, v)) = intermediate_values
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|(s, _)| *s <= rung_step)
|
||||
{
|
||||
v
|
||||
} else {
|
||||
return false;
|
||||
};
|
||||
|
||||
self.is_pruned_at_rung(current_value, rung_step, bracket, completed_trials)
|
||||
}
|
||||
}
|
||||
|
||||
impl HyperbandPruner {
|
||||
/// Determine whether a trial should be pruned at the given rung within its bracket.
|
||||
///
|
||||
/// Only compares against other trials in the same bracket.
|
||||
#[allow(
|
||||
clippy::cast_precision_loss,
|
||||
clippy::cast_possible_truncation,
|
||||
clippy::cast_sign_loss
|
||||
)]
|
||||
fn is_pruned_at_rung(
|
||||
&self,
|
||||
current_value: f64,
|
||||
rung_step: u64,
|
||||
bracket: usize,
|
||||
completed_trials: &[CompletedTrial],
|
||||
) -> bool {
|
||||
let eta = self.reduction_factor as usize;
|
||||
|
||||
// Collect values at this rung step from trials in the same bracket
|
||||
let map = self.trial_brackets.lock().expect("lock poisoned");
|
||||
let mut values_at_rung: Vec<f64> = completed_trials
|
||||
.iter()
|
||||
.filter(|t| t.state == TrialState::Complete || t.state == TrialState::Pruned)
|
||||
.filter(|t| map.get(&t.id).copied() == Some(bracket))
|
||||
.filter_map(|t| {
|
||||
t.intermediate_values
|
||||
.iter()
|
||||
.find(|(s, _)| *s == rung_step)
|
||||
.map(|(_, v)| *v)
|
||||
})
|
||||
.collect();
|
||||
drop(map);
|
||||
|
||||
// Need at least eta trials to make a meaningful comparison
|
||||
if values_at_rung.len() < eta {
|
||||
return false;
|
||||
}
|
||||
|
||||
values_at_rung.push(current_value);
|
||||
|
||||
values_at_rung
|
||||
.sort_unstable_by(|a, b| a.partial_cmp(b).unwrap_or(core::cmp::Ordering::Equal));
|
||||
if self.direction == Direction::Maximize {
|
||||
values_at_rung.reverse();
|
||||
}
|
||||
|
||||
let n_keep = (values_at_rung.len() as f64 / eta as f64).ceil() as usize;
|
||||
let threshold_idx = n_keep.max(1) - 1;
|
||||
let threshold = values_at_rung[threshold_idx];
|
||||
|
||||
match self.direction {
|
||||
Direction::Minimize => current_value > threshold,
|
||||
Direction::Maximize => current_value < threshold,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_trial(id: u64, values: &[(u64, f64)]) -> CompletedTrial {
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::parameter::ParamId;
|
||||
|
||||
CompletedTrial::with_intermediate_values(
|
||||
id,
|
||||
HashMap::<ParamId, crate::ParamValue>::new(),
|
||||
HashMap::new(),
|
||||
HashMap::new(),
|
||||
0.0,
|
||||
values.to_vec(),
|
||||
HashMap::new(),
|
||||
)
|
||||
}
|
||||
|
||||
fn make_pruned_trial(id: u64, values: &[(u64, f64)]) -> CompletedTrial {
|
||||
let mut t = make_trial(id, values);
|
||||
t.state = TrialState::Pruned;
|
||||
t
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s_max_default() {
|
||||
let pruner = HyperbandPruner::new();
|
||||
// s_max = floor(ln(81/1) / ln(3)) = floor(4.0) = 4
|
||||
assert_eq!(pruner.s_max(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s_max_custom() {
|
||||
let pruner = HyperbandPruner::new()
|
||||
.min_resource(1)
|
||||
.max_resource(16)
|
||||
.reduction_factor(2);
|
||||
// s_max = floor(ln(16) / ln(2)) = floor(4.0) = 4
|
||||
assert_eq!(pruner.s_max(), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bracket_count() {
|
||||
let pruner = HyperbandPruner::new();
|
||||
// s_max=4, so brackets 0..=4 → 5 brackets
|
||||
assert_eq!(pruner.s_max() + 1, 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rung_steps_bracket_0_default() {
|
||||
let pruner = HyperbandPruner::new();
|
||||
// Bracket 0: min_resource_bracket = ceil(81 / 3^4) = ceil(81/81) = 1
|
||||
// Rungs: 1, 3, 9, 27, 81
|
||||
assert_eq!(pruner.rung_steps_for_bracket(0), vec![1, 3, 9, 27, 81]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rung_steps_bracket_2_default() {
|
||||
let pruner = HyperbandPruner::new();
|
||||
// Bracket 2: min_resource_bracket = ceil(81 / 3^(4-2)) = ceil(81/9) = 9
|
||||
// Rungs: 9, 27, 81
|
||||
assert_eq!(pruner.rung_steps_for_bracket(2), vec![9, 27, 81]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rung_steps_bracket_4_default() {
|
||||
let pruner = HyperbandPruner::new();
|
||||
// Bracket 4 (s_max): min_resource_bracket = ceil(81 / 3^0) = 81
|
||||
// Rungs: 81 only (no pruning, full budget)
|
||||
assert_eq!(pruner.rung_steps_for_bracket(4), vec![81]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rung_steps_eta2() {
|
||||
let pruner = HyperbandPruner::new()
|
||||
.min_resource(1)
|
||||
.max_resource(16)
|
||||
.reduction_factor(2);
|
||||
// s_max = 4
|
||||
// Bracket 0: min=ceil(16/2^4)=1, rungs: 1,2,4,8,16
|
||||
assert_eq!(pruner.rung_steps_for_bracket(0), vec![1, 2, 4, 8, 16]);
|
||||
// Bracket 2: min=ceil(16/2^2)=4, rungs: 4,8,16
|
||||
assert_eq!(pruner.rung_steps_for_bracket(2), vec![4, 8, 16]);
|
||||
// Bracket 4: min=16, rungs: 16
|
||||
assert_eq!(pruner.rung_steps_for_bracket(4), vec![16]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_robin_bracket_assignment() {
|
||||
let pruner = HyperbandPruner::new(); // 5 brackets (0..=4)
|
||||
// Trials get assigned in round-robin: 0→0, 1→1, 2→2, 3→3, 4→4, 5→0, ...
|
||||
assert_eq!(pruner.assign_bracket(100), 0);
|
||||
assert_eq!(pruner.assign_bracket(101), 1);
|
||||
assert_eq!(pruner.assign_bracket(102), 2);
|
||||
assert_eq!(pruner.assign_bracket(103), 3);
|
||||
assert_eq!(pruner.assign_bracket(104), 4);
|
||||
assert_eq!(pruner.assign_bracket(105), 0); // wraps around
|
||||
|
||||
// Repeated calls for same trial return same bracket
|
||||
assert_eq!(pruner.assign_bracket(100), 0);
|
||||
assert_eq!(pruner.assign_bracket(103), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_prune_before_first_rung() {
|
||||
let pruner = HyperbandPruner::new().direction(Direction::Minimize);
|
||||
// Assign trial 0 to bracket 0 (rungs: 1, 3, 9, 27, 81)
|
||||
pruner.assign_bracket(0);
|
||||
|
||||
// Register completed trials in bracket 0
|
||||
let mut completed = Vec::new();
|
||||
for i in 1..=9 {
|
||||
pruner.assign_bracket(i);
|
||||
completed.push(make_trial(i, &[(1, i as f64)]));
|
||||
}
|
||||
|
||||
// Trial at step 0 (before rung 1) → don't prune
|
||||
assert!(!pruner.should_prune(0, 0, &[(0, 100.0)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_prune_at_max_resource() {
|
||||
let pruner = HyperbandPruner::new().direction(Direction::Minimize);
|
||||
|
||||
// Put all trials in bracket 0
|
||||
let mut completed = Vec::new();
|
||||
for i in 0..9 {
|
||||
pruner.assign_bracket(i);
|
||||
completed.push(make_trial(i, &[(81, (i + 1) as f64)]));
|
||||
}
|
||||
|
||||
let trial_id = 9;
|
||||
pruner.assign_bracket(trial_id);
|
||||
// At max_resource (81), never prune
|
||||
assert!(!pruner.should_prune(trial_id, 81, &[(81, 100.0)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prune_worst_in_bracket_minimize() {
|
||||
let pruner = HyperbandPruner::new().direction(Direction::Minimize);
|
||||
|
||||
// Force all trials into bracket 0 by assigning sequentially
|
||||
// With 5 brackets, trials 0,5,10,... go to bracket 0
|
||||
let bracket_0_ids: Vec<u64> = (0..5).map(|i| i * 5).collect();
|
||||
// Assign all 25 trial IDs to fill brackets
|
||||
for i in 0..25 {
|
||||
pruner.assign_bracket(i);
|
||||
}
|
||||
|
||||
// Create 9 completed trials in bracket 0 at rung step=1
|
||||
let completed: Vec<_> = bracket_0_ids
|
||||
.iter()
|
||||
.take(3)
|
||||
.enumerate()
|
||||
.map(|(idx, &id)| make_trial(id, &[(1, (idx + 1) as f64)]))
|
||||
.collect();
|
||||
|
||||
// Trial 25 → bracket 0 (25 % 5 == 0)
|
||||
let test_id = 25;
|
||||
pruner.assign_bracket(test_id);
|
||||
assert_eq!(pruner.assign_bracket(test_id), 0);
|
||||
|
||||
// 3 completed + 1 current = 4. eta=3. ceil(4/3)=2. Threshold = 2.0
|
||||
// Value 2.0 → keep
|
||||
assert!(!pruner.should_prune(test_id, 1, &[(1, 2.0)], &completed));
|
||||
// Value 3.0 → prune
|
||||
assert!(pruner.should_prune(test_id, 1, &[(1, 3.0)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prune_worst_in_bracket_maximize() {
|
||||
let pruner = HyperbandPruner::new().direction(Direction::Maximize);
|
||||
|
||||
// Assign trials so they end up in bracket 0
|
||||
for i in 0..25 {
|
||||
pruner.assign_bracket(i);
|
||||
}
|
||||
|
||||
let completed: Vec<_> = [0u64, 5, 10]
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(idx, &id)| make_trial(id, &[(1, (idx + 1) as f64)]))
|
||||
.collect();
|
||||
|
||||
let test_id = 25;
|
||||
pruner.assign_bracket(test_id);
|
||||
|
||||
// For maximize, best = highest. Values: 1,2,3 + current
|
||||
// Value 2.0 → keep (threshold = 2.0 when sorted desc: 3,2,current,1)
|
||||
assert!(!pruner.should_prune(test_id, 1, &[(1, 2.0)], &completed));
|
||||
// Value 1.0 → prune
|
||||
assert!(pruner.should_prune(test_id, 1, &[(1, 0.5)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn different_brackets_have_different_aggressiveness() {
|
||||
let pruner = HyperbandPruner::new()
|
||||
.min_resource(1)
|
||||
.max_resource(81)
|
||||
.reduction_factor(3)
|
||||
.direction(Direction::Minimize);
|
||||
|
||||
let rungs_0 = pruner.rung_steps_for_bracket(0);
|
||||
let rungs_2 = pruner.rung_steps_for_bracket(2);
|
||||
let rungs_4 = pruner.rung_steps_for_bracket(4);
|
||||
|
||||
// Bracket 0 has the most rungs (most aggressive)
|
||||
assert!(rungs_0.len() > rungs_2.len());
|
||||
// Bracket 4 has just 1 rung (no pruning)
|
||||
assert_eq!(rungs_4.len(), 1);
|
||||
// Bracket 0 starts earliest
|
||||
assert!(rungs_0[0] < rungs_2[0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trials_in_different_brackets_independent() {
|
||||
let pruner = HyperbandPruner::new().direction(Direction::Minimize);
|
||||
|
||||
// Assign trials: bracket 0 gets IDs 0,5,10,15,20
|
||||
for i in 0..25 {
|
||||
pruner.assign_bracket(i);
|
||||
}
|
||||
|
||||
// Bracket 0 trials: bad values at rung step=1
|
||||
let bracket_0_trials: Vec<_> = [0u64, 5, 10]
|
||||
.iter()
|
||||
.map(|&id| make_trial(id, &[(1, 100.0)]))
|
||||
.collect();
|
||||
|
||||
// Bracket 1 trials: good values at rung step=1
|
||||
let bracket_1_trials: Vec<_> = [1u64, 6, 11]
|
||||
.iter()
|
||||
.map(|&id| make_trial(id, &[(1, 1.0)]))
|
||||
.collect();
|
||||
|
||||
let mut all_trials = bracket_0_trials;
|
||||
all_trials.extend(bracket_1_trials);
|
||||
|
||||
// A new bracket-0 trial with value 50 should be compared against
|
||||
// bracket-0 peers (100,100,100), not bracket-1 peers (1,1,1)
|
||||
let test_id = 25; // bracket 0
|
||||
pruner.assign_bracket(test_id);
|
||||
// 3 peers at 100.0 + current at 50.0. ceil(4/3)=2. Sorted: 50,100,100,100. Threshold=100.0
|
||||
// Value 50.0 < 100.0 → keep
|
||||
assert!(!pruner.should_prune(test_id, 1, &[(1, 50.0)], &all_trials));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn includes_pruned_trials() {
|
||||
let pruner = HyperbandPruner::new().direction(Direction::Minimize);
|
||||
|
||||
for i in 0..25 {
|
||||
pruner.assign_bracket(i);
|
||||
}
|
||||
|
||||
let completed = vec![
|
||||
make_trial(0, &[(1, 1.0)]),
|
||||
make_pruned_trial(5, &[(1, 8.0)]),
|
||||
make_pruned_trial(10, &[(1, 9.0)]),
|
||||
];
|
||||
|
||||
let test_id = 25;
|
||||
pruner.assign_bracket(test_id);
|
||||
|
||||
// Values: 1.0, 8.0, 9.0 + current. eta=3.
|
||||
// Value 1.0 → keep
|
||||
assert!(!pruner.should_prune(test_id, 1, &[(1, 1.0)], &completed));
|
||||
// Value 5.0 → prune (sorted: 1,5,8,9 → keep ceil(4/3)=2 → threshold=5.0, 5.0 not > 5.0 → keep)
|
||||
assert!(!pruner.should_prune(test_id, 1, &[(1, 5.0)], &completed));
|
||||
// Value 6.0 → prune (sorted: 1,6,8,9 → threshold=6.0, 6.0 not > 6.0 → keep)
|
||||
assert!(!pruner.should_prune(test_id, 1, &[(1, 6.0)], &completed));
|
||||
// Value 9.5 → prune
|
||||
assert!(pruner.should_prune(test_id, 1, &[(1, 9.5)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "min_resource must be > 0")]
|
||||
fn rejects_zero_min_resource() {
|
||||
let _ = HyperbandPruner::new().min_resource(0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "max_resource must be > 0")]
|
||||
fn rejects_zero_max_resource() {
|
||||
let _ = HyperbandPruner::new().max_resource(0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "reduction_factor must be >= 2")]
|
||||
fn rejects_reduction_factor_one() {
|
||||
let _ = HyperbandPruner::new().reduction_factor(1);
|
||||
}
|
||||
}
|
||||
@@ -1,127 +0,0 @@
|
||||
use super::Pruner;
|
||||
use super::percentile::compute_percentile;
|
||||
use crate::sampler::CompletedTrial;
|
||||
use crate::types::{Direction, TrialState};
|
||||
|
||||
/// Prune trials that are performing worse than the median of completed trials
|
||||
/// at the same step.
|
||||
///
|
||||
/// This is the most commonly used pruner. It compares the current trial's
|
||||
/// intermediate value at each step with the median of all completed trials'
|
||||
/// values at that same step.
|
||||
///
|
||||
/// Equivalent to `PercentilePruner::new(50.0, direction)`.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::Direction;
|
||||
/// use optimizer::pruner::MedianPruner;
|
||||
///
|
||||
/// // Prune trials worse than median when minimizing, after 5 warmup steps
|
||||
/// let pruner = MedianPruner::new(Direction::Minimize)
|
||||
/// .n_warmup_steps(5)
|
||||
/// .n_min_trials(3);
|
||||
/// ```
|
||||
pub struct MedianPruner {
|
||||
/// The optimization direction.
|
||||
direction: Direction,
|
||||
/// Don't prune in the first N steps (let the trial warm up).
|
||||
n_warmup_steps: u64,
|
||||
/// Require at least N completed trials before pruning.
|
||||
n_min_trials: usize,
|
||||
}
|
||||
|
||||
impl MedianPruner {
|
||||
/// Create a new `MedianPruner` for the given optimization direction.
|
||||
///
|
||||
/// By default, `n_warmup_steps` is 0 and `n_min_trials` is 1.
|
||||
#[must_use]
|
||||
pub fn new(direction: Direction) -> Self {
|
||||
Self {
|
||||
direction,
|
||||
n_warmup_steps: 0,
|
||||
n_min_trials: 1,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the number of warmup steps. No pruning occurs before this step.
|
||||
#[must_use]
|
||||
pub fn n_warmup_steps(mut self, n: u64) -> Self {
|
||||
self.n_warmup_steps = n;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the minimum number of completed trials required before pruning.
|
||||
#[must_use]
|
||||
pub fn n_min_trials(mut self, n: usize) -> Self {
|
||||
self.n_min_trials = n;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Pruner for MedianPruner {
|
||||
fn should_prune(
|
||||
&self,
|
||||
_trial_id: u64,
|
||||
step: u64,
|
||||
intermediate_values: &[(u64, f64)],
|
||||
completed_trials: &[CompletedTrial],
|
||||
) -> bool {
|
||||
// 1. Don't prune during warmup
|
||||
if step < self.n_warmup_steps {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Get the current trial's latest value
|
||||
let Some(&(_, current_value)) = intermediate_values.last() else {
|
||||
return false;
|
||||
};
|
||||
|
||||
// 2. Collect values at this step from completed (non-pruned) trials
|
||||
let mut values_at_step: Vec<f64> = completed_trials
|
||||
.iter()
|
||||
.filter(|t| t.state == TrialState::Complete)
|
||||
.filter_map(|t| {
|
||||
t.intermediate_values
|
||||
.iter()
|
||||
.find(|(s, _)| *s == step)
|
||||
.map(|(_, v)| *v)
|
||||
})
|
||||
.collect();
|
||||
|
||||
// 3. Not enough trials
|
||||
if values_at_step.len() < self.n_min_trials {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 4. Compute median (50th percentile)
|
||||
let median = compute_percentile(&mut values_at_step, 50.0);
|
||||
|
||||
// 5. Compare against median based on direction
|
||||
match self.direction {
|
||||
Direction::Minimize => current_value > median,
|
||||
Direction::Maximize => current_value < median,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn compute_median_odd() {
|
||||
assert!((compute_percentile(&mut [3.0, 1.0, 2.0], 50.0) - 2.0).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_median_even() {
|
||||
assert!((compute_percentile(&mut [4.0, 1.0, 3.0, 2.0], 50.0) - 2.5).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_median_single() {
|
||||
assert!((compute_percentile(&mut [5.0], 50.0) - 5.0).abs() < f64::EPSILON);
|
||||
}
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
//! Pruner trait and implementations for trial pruning.
|
||||
//!
|
||||
//! Pruners decide whether to stop (prune) a trial early based on its
|
||||
//! intermediate values compared to other trials. This is useful for
|
||||
//! discarding unpromising trials before they complete, saving compute.
|
||||
|
||||
mod hyperband;
|
||||
mod median;
|
||||
mod nop;
|
||||
mod patient;
|
||||
pub(crate) mod percentile;
|
||||
mod successive_halving;
|
||||
mod threshold;
|
||||
mod wilcoxon;
|
||||
|
||||
pub use hyperband::HyperbandPruner;
|
||||
pub use median::MedianPruner;
|
||||
pub use nop::NopPruner;
|
||||
pub use patient::PatientPruner;
|
||||
pub use percentile::PercentilePruner;
|
||||
pub use successive_halving::SuccessiveHalvingPruner;
|
||||
pub use threshold::ThresholdPruner;
|
||||
pub use wilcoxon::WilcoxonPruner;
|
||||
|
||||
use crate::sampler::CompletedTrial;
|
||||
|
||||
/// Trait for pluggable trial pruning strategies.
|
||||
///
|
||||
/// Pruners are consulted after each intermediate value is reported to
|
||||
/// decide whether the trial should be stopped early. The trait requires
|
||||
/// `Send + Sync` to support concurrent and async optimization.
|
||||
///
|
||||
/// # Implementing a custom pruner
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::pruner::Pruner;
|
||||
/// use optimizer::sampler::CompletedTrial;
|
||||
///
|
||||
/// struct MyPruner {
|
||||
/// threshold: f64,
|
||||
/// }
|
||||
///
|
||||
/// impl Pruner for MyPruner {
|
||||
/// fn should_prune(
|
||||
/// &self,
|
||||
/// _trial_id: u64,
|
||||
/// _step: u64,
|
||||
/// intermediate_values: &[(u64, f64)],
|
||||
/// _completed_trials: &[CompletedTrial],
|
||||
/// ) -> bool {
|
||||
/// // Prune if the latest value exceeds the threshold
|
||||
/// intermediate_values
|
||||
/// .last()
|
||||
/// .is_some_and(|&(_, v)| v > self.threshold)
|
||||
/// }
|
||||
/// }
|
||||
/// ```
|
||||
pub trait Pruner: Send + Sync {
|
||||
/// Decide whether to prune a trial at the given step.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `trial_id` - The current trial's ID.
|
||||
/// * `step` - The step at which the intermediate value was reported.
|
||||
/// * `intermediate_values` - All `(step, value)` pairs reported so far for this trial.
|
||||
/// * `completed_trials` - History of all completed trials (for comparison).
|
||||
fn should_prune(
|
||||
&self,
|
||||
trial_id: u64,
|
||||
step: u64,
|
||||
intermediate_values: &[(u64, f64)],
|
||||
completed_trials: &[CompletedTrial],
|
||||
) -> bool;
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
use super::Pruner;
|
||||
use crate::sampler::CompletedTrial;
|
||||
|
||||
/// A pruner that never prunes. This is the default when no pruner is configured.
|
||||
pub struct NopPruner;
|
||||
|
||||
impl Pruner for NopPruner {
|
||||
fn should_prune(
|
||||
&self,
|
||||
_trial_id: u64,
|
||||
_step: u64,
|
||||
_intermediate_values: &[(u64, f64)],
|
||||
_completed_trials: &[CompletedTrial],
|
||||
) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Mutex;
|
||||
|
||||
use super::Pruner;
|
||||
use crate::sampler::CompletedTrial;
|
||||
|
||||
/// Wraps another pruner and adds a patience window.
|
||||
///
|
||||
/// The inner pruner must recommend pruning for `patience` consecutive
|
||||
/// steps before this pruner actually prunes the trial. This is useful
|
||||
/// to prevent premature pruning when intermediate values are noisy.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::pruner::{PatientPruner, ThresholdPruner};
|
||||
///
|
||||
/// // Only prune after the threshold pruner recommends pruning 3 times in a row
|
||||
/// let inner = ThresholdPruner::new().upper(100.0);
|
||||
/// let pruner = PatientPruner::new(inner, 3);
|
||||
/// ```
|
||||
pub struct PatientPruner {
|
||||
inner: Box<dyn Pruner>,
|
||||
patience: u64,
|
||||
/// Track consecutive prune recommendations per trial.
|
||||
consecutive_counts: Mutex<HashMap<u64, u64>>,
|
||||
}
|
||||
|
||||
impl PatientPruner {
|
||||
/// Create a new `PatientPruner` wrapping the given inner pruner.
|
||||
///
|
||||
/// The inner pruner must recommend pruning for `patience` consecutive
|
||||
/// calls before this pruner returns `true`.
|
||||
pub fn new(inner: impl Pruner + 'static, patience: u64) -> Self {
|
||||
Self {
|
||||
inner: Box::new(inner),
|
||||
patience,
|
||||
consecutive_counts: Mutex::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Pruner for PatientPruner {
|
||||
fn should_prune(
|
||||
&self,
|
||||
trial_id: u64,
|
||||
step: u64,
|
||||
intermediate_values: &[(u64, f64)],
|
||||
completed_trials: &[CompletedTrial],
|
||||
) -> bool {
|
||||
let inner_says_prune =
|
||||
self.inner
|
||||
.should_prune(trial_id, step, intermediate_values, completed_trials);
|
||||
let mut counts = self.consecutive_counts.lock().expect("lock poisoned");
|
||||
let count = counts.entry(trial_id).or_insert(0);
|
||||
if inner_says_prune {
|
||||
*count += 1;
|
||||
*count >= self.patience
|
||||
} else {
|
||||
*count = 0;
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::pruner::ThresholdPruner;
|
||||
|
||||
/// A test pruner that always returns the given value.
|
||||
struct ConstPruner(bool);
|
||||
|
||||
impl Pruner for ConstPruner {
|
||||
fn should_prune(
|
||||
&self,
|
||||
_trial_id: u64,
|
||||
_step: u64,
|
||||
_intermediate_values: &[(u64, f64)],
|
||||
_completed_trials: &[CompletedTrial],
|
||||
) -> bool {
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
/// A pruner that returns values from a sequence.
|
||||
struct SequencePruner(Mutex<Vec<bool>>);
|
||||
|
||||
impl Pruner for SequencePruner {
|
||||
fn should_prune(
|
||||
&self,
|
||||
_trial_id: u64,
|
||||
_step: u64,
|
||||
_intermediate_values: &[(u64, f64)],
|
||||
_completed_trials: &[CompletedTrial],
|
||||
) -> bool {
|
||||
self.0.lock().expect("lock poisoned").remove(0)
|
||||
}
|
||||
}
|
||||
|
||||
fn call(pruner: &PatientPruner, trial_id: u64, step: u64) -> bool {
|
||||
pruner.should_prune(trial_id, step, &[(step, 0.0)], &[])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn patience_1_behaves_like_inner() {
|
||||
let pruner = PatientPruner::new(ConstPruner(true), 1);
|
||||
assert!(call(&pruner, 0, 0));
|
||||
assert!(call(&pruner, 0, 1));
|
||||
|
||||
let pruner = PatientPruner::new(ConstPruner(false), 1);
|
||||
assert!(!call(&pruner, 0, 0));
|
||||
assert!(!call(&pruner, 0, 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn patience_3_requires_consecutive_recommendations() {
|
||||
let pruner = PatientPruner::new(ConstPruner(true), 3);
|
||||
assert!(!call(&pruner, 0, 0)); // count=1
|
||||
assert!(!call(&pruner, 0, 1)); // count=2
|
||||
assert!(call(&pruner, 0, 2)); // count=3 → prune
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn counter_resets_on_no_prune() {
|
||||
// Sequence: prune, prune, no-prune, prune, prune, prune
|
||||
let seq = vec![true, true, false, true, true, true];
|
||||
let pruner = PatientPruner::new(SequencePruner(Mutex::new(seq)), 3);
|
||||
|
||||
assert!(!call(&pruner, 0, 0)); // count=1
|
||||
assert!(!call(&pruner, 0, 1)); // count=2
|
||||
assert!(!call(&pruner, 0, 2)); // reset → count=0
|
||||
assert!(!call(&pruner, 0, 3)); // count=1
|
||||
assert!(!call(&pruner, 0, 4)); // count=2
|
||||
assert!(call(&pruner, 0, 5)); // count=3 → prune
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn independent_per_trial() {
|
||||
let pruner = PatientPruner::new(ConstPruner(true), 2);
|
||||
assert!(!call(&pruner, 0, 0)); // trial 0: count=1
|
||||
assert!(!call(&pruner, 1, 0)); // trial 1: count=1
|
||||
assert!(call(&pruner, 0, 1)); // trial 0: count=2 → prune
|
||||
assert!(!call(&pruner, 2, 0)); // trial 2: count=1
|
||||
assert!(call(&pruner, 1, 1)); // trial 1: count=2 → prune
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn works_with_threshold_pruner() {
|
||||
let inner = ThresholdPruner::new().upper(10.0);
|
||||
let pruner = PatientPruner::new(inner, 2);
|
||||
|
||||
// Value below threshold → inner says no
|
||||
assert!(!pruner.should_prune(0, 0, &[(0, 5.0)], &[]));
|
||||
// Value above threshold → inner says yes, count=1
|
||||
assert!(!pruner.should_prune(0, 1, &[(0, 5.0), (1, 15.0)], &[]));
|
||||
// Value above threshold again → count=2 → prune
|
||||
assert!(pruner.should_prune(0, 2, &[(0, 5.0), (1, 15.0), (2, 20.0)], &[]));
|
||||
}
|
||||
}
|
||||
@@ -1,294 +0,0 @@
|
||||
use super::Pruner;
|
||||
use crate::sampler::CompletedTrial;
|
||||
use crate::types::{Direction, TrialState};
|
||||
|
||||
/// Prune trials that are not in the top `percentile`% of completed trials
|
||||
/// at the same training step.
|
||||
///
|
||||
/// `PercentilePruner::new(50.0, direction)` is equivalent to `MedianPruner`.
|
||||
/// `PercentilePruner::new(25.0, direction)` keeps only the top 25% of trials.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::Direction;
|
||||
/// use optimizer::pruner::PercentilePruner;
|
||||
///
|
||||
/// // Keep only the top 25% of trials (aggressive pruning)
|
||||
/// let pruner = PercentilePruner::new(25.0, Direction::Minimize)
|
||||
/// .n_warmup_steps(5)
|
||||
/// .n_min_trials(3);
|
||||
/// ```
|
||||
pub struct PercentilePruner {
|
||||
/// Keep trials in the top `percentile`%. Range: (0.0, 100.0).
|
||||
percentile: f64,
|
||||
/// Don't prune in the first N steps (let the trial warm up).
|
||||
n_warmup_steps: u64,
|
||||
/// Require at least N completed trials before pruning.
|
||||
n_min_trials: usize,
|
||||
/// The optimization direction.
|
||||
direction: Direction,
|
||||
}
|
||||
|
||||
impl PercentilePruner {
|
||||
/// Create a new `PercentilePruner` for the given percentile and direction.
|
||||
///
|
||||
/// The `percentile` value must be in `(0.0, 100.0)`.
|
||||
/// A percentile of 50.0 is equivalent to median pruning.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `percentile` is not in `(0.0, 100.0)`.
|
||||
#[must_use]
|
||||
pub fn new(percentile: f64, direction: Direction) -> Self {
|
||||
assert!(
|
||||
percentile > 0.0 && percentile < 100.0,
|
||||
"percentile must be in (0.0, 100.0), got {percentile}"
|
||||
);
|
||||
Self {
|
||||
percentile,
|
||||
n_warmup_steps: 0,
|
||||
n_min_trials: 1,
|
||||
direction,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the number of warmup steps. No pruning occurs before this step.
|
||||
#[must_use]
|
||||
pub fn n_warmup_steps(mut self, n: u64) -> Self {
|
||||
self.n_warmup_steps = n;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the minimum number of completed trials required before pruning.
|
||||
#[must_use]
|
||||
pub fn n_min_trials(mut self, n: usize) -> Self {
|
||||
self.n_min_trials = n;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Pruner for PercentilePruner {
|
||||
fn should_prune(
|
||||
&self,
|
||||
_trial_id: u64,
|
||||
step: u64,
|
||||
intermediate_values: &[(u64, f64)],
|
||||
completed_trials: &[CompletedTrial],
|
||||
) -> bool {
|
||||
// 1. Don't prune during warmup
|
||||
if step < self.n_warmup_steps {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Get the current trial's latest value
|
||||
let Some(&(_, current_value)) = intermediate_values.last() else {
|
||||
return false;
|
||||
};
|
||||
|
||||
// 2. Collect values at this step from completed (non-pruned) trials
|
||||
let mut values_at_step: Vec<f64> = completed_trials
|
||||
.iter()
|
||||
.filter(|t| t.state == TrialState::Complete)
|
||||
.filter_map(|t| {
|
||||
t.intermediate_values
|
||||
.iter()
|
||||
.find(|(s, _)| *s == step)
|
||||
.map(|(_, v)| *v)
|
||||
})
|
||||
.collect();
|
||||
|
||||
// 3. Not enough trials
|
||||
if values_at_step.len() < self.n_min_trials {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 4. Compute percentile threshold
|
||||
let threshold = compute_percentile(&mut values_at_step, self.percentile);
|
||||
|
||||
// 5. Compare against threshold based on direction
|
||||
match self.direction {
|
||||
Direction::Minimize => current_value > threshold,
|
||||
Direction::Maximize => current_value < threshold,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the given percentile of a non-empty slice. Sorts the slice in place.
|
||||
///
|
||||
/// Uses linear interpolation between the two nearest ranks.
|
||||
#[allow(
|
||||
clippy::cast_precision_loss,
|
||||
clippy::cast_possible_truncation,
|
||||
clippy::cast_sign_loss
|
||||
)]
|
||||
pub(crate) fn compute_percentile(values: &mut [f64], percentile: f64) -> f64 {
|
||||
values.sort_unstable_by(|a, b| a.partial_cmp(b).unwrap_or(core::cmp::Ordering::Equal));
|
||||
let len = values.len();
|
||||
if len == 1 {
|
||||
return values[0];
|
||||
}
|
||||
// Rank in [0, len-1] range
|
||||
let rank = percentile / 100.0 * (len - 1) as f64;
|
||||
let lower = rank.floor() as usize;
|
||||
let upper = rank.ceil() as usize;
|
||||
if lower == upper {
|
||||
values[lower]
|
||||
} else {
|
||||
let frac = rank - lower as f64;
|
||||
values[lower] * (1.0 - frac) + values[upper] * frac
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn compute_percentile_median_odd() {
|
||||
// Percentile 50 on odd-length slice = median
|
||||
let val = compute_percentile(&mut [3.0, 1.0, 2.0], 50.0);
|
||||
assert!((val - 2.0).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_percentile_median_even() {
|
||||
// Percentile 50 on even-length slice = median (interpolated)
|
||||
let val = compute_percentile(&mut [4.0, 1.0, 3.0, 2.0], 50.0);
|
||||
assert!((val - 2.5).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_percentile_25() {
|
||||
// [1.0, 2.0, 3.0, 4.0], rank = 0.25 * 3 = 0.75
|
||||
// interpolate: 1.0 * 0.25 + 2.0 * 0.75 = 1.75
|
||||
let val = compute_percentile(&mut [4.0, 1.0, 3.0, 2.0], 25.0);
|
||||
assert!((val - 1.75).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_percentile_75() {
|
||||
// [1.0, 2.0, 3.0, 4.0], rank = 0.75 * 3 = 2.25
|
||||
// interpolate: 3.0 * 0.75 + 4.0 * 0.25 = 3.25
|
||||
let val = compute_percentile(&mut [4.0, 1.0, 3.0, 2.0], 75.0);
|
||||
assert!((val - 3.25).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_percentile_single() {
|
||||
let val = compute_percentile(&mut [5.0], 50.0);
|
||||
assert!((val - 5.0).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "percentile must be in (0.0, 100.0)")]
|
||||
fn new_rejects_zero() {
|
||||
let _ = PercentilePruner::new(0.0, Direction::Minimize);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "percentile must be in (0.0, 100.0)")]
|
||||
fn new_rejects_hundred() {
|
||||
let _ = PercentilePruner::new(100.0, Direction::Minimize);
|
||||
}
|
||||
|
||||
fn make_completed_trial(id: u64, values: &[(u64, f64)]) -> CompletedTrial {
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::parameter::ParamId;
|
||||
|
||||
CompletedTrial::with_intermediate_values(
|
||||
id,
|
||||
HashMap::<ParamId, crate::ParamValue>::new(),
|
||||
HashMap::new(),
|
||||
HashMap::new(),
|
||||
0.0,
|
||||
values.to_vec(),
|
||||
HashMap::new(),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn percentile_50_matches_median_behavior() {
|
||||
let pruner = PercentilePruner::new(50.0, Direction::Minimize);
|
||||
let completed = vec![
|
||||
make_completed_trial(0, &[(0, 1.0), (1, 2.0)]),
|
||||
make_completed_trial(1, &[(0, 3.0), (1, 4.0)]),
|
||||
make_completed_trial(2, &[(0, 5.0), (1, 6.0)]),
|
||||
];
|
||||
// Median at step 1 is 4.0
|
||||
// Value 5.0 > 4.0 → prune
|
||||
assert!(pruner.should_prune(3, 1, &[(0, 3.0), (1, 5.0)], &completed));
|
||||
// Value 3.0 < 4.0 → keep
|
||||
assert!(!pruner.should_prune(3, 1, &[(0, 3.0), (1, 3.0)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn percentile_25_is_more_aggressive() {
|
||||
let pruner_25 = PercentilePruner::new(25.0, Direction::Minimize);
|
||||
let pruner_75 = PercentilePruner::new(75.0, Direction::Minimize);
|
||||
let completed = vec![
|
||||
make_completed_trial(0, &[(0, 1.0)]),
|
||||
make_completed_trial(1, &[(0, 2.0)]),
|
||||
make_completed_trial(2, &[(0, 3.0)]),
|
||||
make_completed_trial(3, &[(0, 4.0)]),
|
||||
];
|
||||
// 25th percentile at step 0: 1.75
|
||||
// 75th percentile at step 0: 3.25
|
||||
// Value 2.5: above 25th (prune), below 75th (keep)
|
||||
assert!(pruner_25.should_prune(4, 0, &[(0, 2.5)], &completed));
|
||||
assert!(!pruner_75.should_prune(4, 0, &[(0, 2.5)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn warmup_prevents_pruning() {
|
||||
let pruner = PercentilePruner::new(50.0, Direction::Minimize).n_warmup_steps(5);
|
||||
let completed = vec![make_completed_trial(0, &[(0, 1.0)])];
|
||||
// Step 3 < warmup 5 → no prune even with bad value
|
||||
assert!(!pruner.should_prune(1, 3, &[(3, 100.0)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn n_min_trials_prevents_pruning() {
|
||||
let pruner = PercentilePruner::new(50.0, Direction::Minimize).n_min_trials(5);
|
||||
let completed = vec![
|
||||
make_completed_trial(0, &[(0, 1.0)]),
|
||||
make_completed_trial(1, &[(0, 2.0)]),
|
||||
];
|
||||
// Only 2 trials, need 5 → no prune
|
||||
assert!(!pruner.should_prune(2, 0, &[(0, 100.0)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maximize_direction() {
|
||||
let pruner = PercentilePruner::new(50.0, Direction::Maximize);
|
||||
let completed = vec![
|
||||
make_completed_trial(0, &[(0, 1.0)]),
|
||||
make_completed_trial(1, &[(0, 3.0)]),
|
||||
make_completed_trial(2, &[(0, 5.0)]),
|
||||
];
|
||||
// Median at step 0 is 3.0
|
||||
// Value 2.0 < 3.0 → prune (maximize wants higher)
|
||||
assert!(pruner.should_prune(3, 0, &[(0, 2.0)], &completed));
|
||||
// Value 4.0 > 3.0 → keep
|
||||
assert!(!pruner.should_prune(3, 0, &[(0, 4.0)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn near_boundary_percentiles() {
|
||||
let pruner_low = PercentilePruner::new(1.0, Direction::Minimize);
|
||||
let pruner_high = PercentilePruner::new(99.0, Direction::Minimize);
|
||||
let completed = vec![
|
||||
make_completed_trial(0, &[(0, 1.0)]),
|
||||
make_completed_trial(1, &[(0, 2.0)]),
|
||||
make_completed_trial(2, &[(0, 3.0)]),
|
||||
make_completed_trial(3, &[(0, 100.0)]),
|
||||
];
|
||||
// Percentile 1 is very aggressive (threshold near 1.0)
|
||||
// Value 1.5 should be pruned
|
||||
assert!(pruner_low.should_prune(4, 0, &[(0, 1.5)], &completed));
|
||||
// Percentile 99 is very lenient (threshold near 100.0)
|
||||
// Value 50.0 should not be pruned
|
||||
assert!(!pruner_high.should_prune(4, 0, &[(0, 50.0)], &completed));
|
||||
}
|
||||
}
|
||||
@@ -1,466 +0,0 @@
|
||||
use super::Pruner;
|
||||
use crate::sampler::CompletedTrial;
|
||||
use crate::types::{Direction, TrialState};
|
||||
|
||||
/// Successive Halving pruner based on the SHA algorithm.
|
||||
///
|
||||
/// Trials are evaluated at exponentially-spaced "rungs". At each rung,
|
||||
/// only the top 1/eta fraction of trials survive to the next rung.
|
||||
///
|
||||
/// For example, with `min_resource=1`, `max_resource=81`, `reduction_factor=3`:
|
||||
/// - Rung 0: evaluate at step 1, keep top 1/3
|
||||
/// - Rung 1: evaluate at step 3, keep top 1/3
|
||||
/// - Rung 2: evaluate at step 9, keep top 1/3
|
||||
/// - Rung 3: evaluate at step 27, keep top 1/3
|
||||
/// - Rung 4: evaluate at step 81 (full budget)
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::Direction;
|
||||
/// use optimizer::pruner::SuccessiveHalvingPruner;
|
||||
///
|
||||
/// let pruner = SuccessiveHalvingPruner::new()
|
||||
/// .min_resource(1)
|
||||
/// .max_resource(81)
|
||||
/// .reduction_factor(3)
|
||||
/// .direction(Direction::Minimize);
|
||||
/// ```
|
||||
pub struct SuccessiveHalvingPruner {
|
||||
min_resource: u64,
|
||||
max_resource: u64,
|
||||
reduction_factor: u64,
|
||||
min_early_stopping_rate: u64,
|
||||
direction: Direction,
|
||||
}
|
||||
|
||||
impl SuccessiveHalvingPruner {
|
||||
/// Create a new `SuccessiveHalvingPruner` with default parameters.
|
||||
///
|
||||
/// Defaults: `min_resource=1`, `max_resource=81`, `reduction_factor=3`,
|
||||
/// `min_early_stopping_rate=0`, `direction=Minimize`.
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
min_resource: 1,
|
||||
max_resource: 81,
|
||||
reduction_factor: 3,
|
||||
min_early_stopping_rate: 0,
|
||||
direction: Direction::Minimize,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the minimum resource (budget) per trial.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `r` is 0.
|
||||
#[must_use]
|
||||
pub fn min_resource(mut self, r: u64) -> Self {
|
||||
assert!(r > 0, "min_resource must be > 0, got {r}");
|
||||
self.min_resource = r;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the maximum resource (budget) per trial.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `r` is 0.
|
||||
#[must_use]
|
||||
pub fn max_resource(mut self, r: u64) -> Self {
|
||||
assert!(r > 0, "max_resource must be > 0, got {r}");
|
||||
self.max_resource = r;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the reduction factor (eta). At each rung, the top 1/eta trials survive.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `eta` is less than 2.
|
||||
#[must_use]
|
||||
pub fn reduction_factor(mut self, eta: u64) -> Self {
|
||||
assert!(eta >= 2, "reduction_factor must be >= 2, got {eta}");
|
||||
self.reduction_factor = eta;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the minimum early stopping rate. Skips the first N rungs.
|
||||
#[must_use]
|
||||
pub fn min_early_stopping_rate(mut self, n: u64) -> Self {
|
||||
self.min_early_stopping_rate = n;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the optimization direction.
|
||||
#[must_use]
|
||||
pub fn direction(mut self, d: Direction) -> Self {
|
||||
self.direction = d;
|
||||
self
|
||||
}
|
||||
|
||||
/// Compute the rung steps: `[min_resource * eta^(s), ...]` up to `max_resource`,
|
||||
/// skipping the first `min_early_stopping_rate` rungs.
|
||||
fn rung_steps(&self) -> Vec<u64> {
|
||||
let eta = self.reduction_factor;
|
||||
let mut steps = Vec::new();
|
||||
let mut rung: u32 = 0;
|
||||
while let Some(power) = eta.checked_pow(rung) {
|
||||
let step = self.min_resource.saturating_mul(power);
|
||||
if step > self.max_resource {
|
||||
break;
|
||||
}
|
||||
if u64::from(rung) >= self.min_early_stopping_rate {
|
||||
steps.push(step);
|
||||
}
|
||||
rung += 1;
|
||||
}
|
||||
steps
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SuccessiveHalvingPruner {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
impl Pruner for SuccessiveHalvingPruner {
|
||||
fn should_prune(
|
||||
&self,
|
||||
_trial_id: u64,
|
||||
step: u64,
|
||||
intermediate_values: &[(u64, f64)],
|
||||
completed_trials: &[CompletedTrial],
|
||||
) -> bool {
|
||||
let rungs = self.rung_steps();
|
||||
|
||||
// Find the highest rung step <= current step
|
||||
let Some(&rung_step) = rungs.iter().rev().find(|&&r| r <= step) else {
|
||||
// No rung matches (before the first rung) → don't prune
|
||||
return false;
|
||||
};
|
||||
|
||||
// If this is the last rung (full budget), don't prune
|
||||
if rung_step >= self.max_resource {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Get the current trial's value at this rung step
|
||||
let Some(&(_, current_value)) = intermediate_values.iter().find(|(s, _)| *s == rung_step)
|
||||
else {
|
||||
// Trial hasn't reported a value at this exact rung step.
|
||||
// Use the latest intermediate value at or before the rung step instead.
|
||||
let Some(&(_, current_value)) = intermediate_values
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|(s, _)| *s <= rung_step)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
return self.is_pruned_at_rung(current_value, rung_step, completed_trials);
|
||||
};
|
||||
|
||||
self.is_pruned_at_rung(current_value, rung_step, completed_trials)
|
||||
}
|
||||
}
|
||||
|
||||
impl SuccessiveHalvingPruner {
|
||||
/// Determine whether a trial with `current_value` should be pruned at the given rung.
|
||||
#[allow(
|
||||
clippy::cast_precision_loss,
|
||||
clippy::cast_possible_truncation,
|
||||
clippy::cast_sign_loss
|
||||
)]
|
||||
fn is_pruned_at_rung(
|
||||
&self,
|
||||
current_value: f64,
|
||||
rung_step: u64,
|
||||
completed_trials: &[CompletedTrial],
|
||||
) -> bool {
|
||||
let eta = self.reduction_factor as usize;
|
||||
|
||||
// Collect values at this rung step from all trials that reached it
|
||||
let mut values_at_rung: Vec<f64> = completed_trials
|
||||
.iter()
|
||||
.filter(|t| t.state == TrialState::Complete || t.state == TrialState::Pruned)
|
||||
.filter_map(|t| {
|
||||
t.intermediate_values
|
||||
.iter()
|
||||
.find(|(s, _)| *s == rung_step)
|
||||
.map(|(_, v)| *v)
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Need at least eta trials to make a meaningful comparison
|
||||
// (with fewer trials, we can't determine the top 1/eta fraction)
|
||||
if values_at_rung.len() < eta {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Include the current trial's value for ranking
|
||||
values_at_rung.push(current_value);
|
||||
|
||||
// Sort based on direction: best values first
|
||||
values_at_rung
|
||||
.sort_unstable_by(|a, b| a.partial_cmp(b).unwrap_or(core::cmp::Ordering::Equal));
|
||||
if self.direction == Direction::Maximize {
|
||||
values_at_rung.reverse();
|
||||
}
|
||||
|
||||
// Keep top 1/eta fraction
|
||||
let n_keep = (values_at_rung.len() as f64 / eta as f64).ceil() as usize;
|
||||
let threshold_idx = n_keep.max(1) - 1;
|
||||
let threshold = values_at_rung[threshold_idx];
|
||||
|
||||
// Prune if current value is worse than the threshold
|
||||
match self.direction {
|
||||
Direction::Minimize => current_value > threshold,
|
||||
Direction::Maximize => current_value < threshold,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_trial(id: u64, values: &[(u64, f64)]) -> CompletedTrial {
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::parameter::ParamId;
|
||||
|
||||
CompletedTrial::with_intermediate_values(
|
||||
id,
|
||||
HashMap::<ParamId, crate::ParamValue>::new(),
|
||||
HashMap::new(),
|
||||
HashMap::new(),
|
||||
0.0,
|
||||
values.to_vec(),
|
||||
HashMap::new(),
|
||||
)
|
||||
}
|
||||
|
||||
fn make_pruned_trial(id: u64, values: &[(u64, f64)]) -> CompletedTrial {
|
||||
let mut t = make_trial(id, values);
|
||||
t.state = TrialState::Pruned;
|
||||
t
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rung_steps_default() {
|
||||
let pruner = SuccessiveHalvingPruner::new();
|
||||
let rungs = pruner.rung_steps();
|
||||
// min=1, max=81, eta=3 → 1, 3, 9, 27, 81
|
||||
assert_eq!(rungs, vec![1, 3, 9, 27, 81]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rung_steps_custom() {
|
||||
let pruner = SuccessiveHalvingPruner::new()
|
||||
.min_resource(2)
|
||||
.max_resource(32)
|
||||
.reduction_factor(2);
|
||||
let rungs = pruner.rung_steps();
|
||||
// 2, 4, 8, 16, 32
|
||||
assert_eq!(rungs, vec![2, 4, 8, 16, 32]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rung_steps_with_early_stopping_rate() {
|
||||
let pruner = SuccessiveHalvingPruner::new().min_early_stopping_rate(2);
|
||||
let rungs = pruner.rung_steps();
|
||||
// Skip rung 0 (step=1) and rung 1 (step=3), keep rung 2+ (9, 27, 81)
|
||||
assert_eq!(rungs, vec![9, 27, 81]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_prune_before_first_rung() {
|
||||
let pruner = SuccessiveHalvingPruner::new()
|
||||
.min_resource(10)
|
||||
.max_resource(100)
|
||||
.reduction_factor(3);
|
||||
let completed = vec![
|
||||
make_trial(0, &[(5, 1.0)]),
|
||||
make_trial(1, &[(5, 2.0)]),
|
||||
make_trial(2, &[(5, 3.0)]),
|
||||
];
|
||||
// Step 5 is before the first rung (10)
|
||||
assert!(!pruner.should_prune(3, 5, &[(5, 100.0)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_prune_with_single_trial() {
|
||||
let pruner = SuccessiveHalvingPruner::new();
|
||||
let completed = vec![make_trial(0, &[(1, 5.0)]), make_trial(1, &[(1, 3.0)])];
|
||||
// Only 2 completed trials at rung + 1 current = 3 total, threshold = ceil(3/3) = 1
|
||||
// With eta=3, we need at least 3 completed trials
|
||||
assert!(!pruner.should_prune(2, 1, &[(1, 10.0)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prune_worst_trials_at_rung() {
|
||||
let pruner = SuccessiveHalvingPruner::new().direction(Direction::Minimize);
|
||||
|
||||
// 9 completed trials at rung step=1, with values 1..=9
|
||||
let completed: Vec<_> = (0..9)
|
||||
.map(|i| make_trial(i, &[(1, (i + 1) as f64)]))
|
||||
.collect();
|
||||
|
||||
// With eta=3, keep top 1/3. 10 total values → ceil(10/3) = 4 kept
|
||||
// Best 4 values: 1, 2, 3, 4. Threshold = 4.0
|
||||
// Value 3.0 → keep (in top 1/3)
|
||||
assert!(!pruner.should_prune(9, 1, &[(1, 3.0)], &completed));
|
||||
// Value 5.0 → prune (not in top 1/3)
|
||||
assert!(pruner.should_prune(9, 1, &[(1, 5.0)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn top_fraction_survives() {
|
||||
let pruner = SuccessiveHalvingPruner::new().direction(Direction::Minimize);
|
||||
|
||||
// 6 completed trials at step=1
|
||||
let completed: Vec<_> = (0..6)
|
||||
.map(|i| make_trial(i, &[(1, (i + 1) as f64)]))
|
||||
.collect();
|
||||
|
||||
// 7 total (6 + current). ceil(7/3) = 3 keep. Threshold = 3.0
|
||||
// Value 2.0 → keep
|
||||
assert!(!pruner.should_prune(6, 1, &[(1, 2.0)], &completed));
|
||||
// Value 3.0 → keep (at threshold)
|
||||
assert!(!pruner.should_prune(6, 1, &[(1, 3.0)], &completed));
|
||||
// Value 4.0 → prune
|
||||
assert!(pruner.should_prune(6, 1, &[(1, 4.0)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maximize_direction() {
|
||||
let pruner = SuccessiveHalvingPruner::new().direction(Direction::Maximize);
|
||||
|
||||
let completed: Vec<_> = (0..6)
|
||||
.map(|i| make_trial(i, &[(1, (i + 1) as f64)]))
|
||||
.collect();
|
||||
|
||||
// 7 total. For maximize, best = highest. ceil(7/3)=3. Top 3: 6,5,4. Threshold=4.0
|
||||
// Value 5.0 → keep
|
||||
assert!(!pruner.should_prune(6, 1, &[(1, 5.0)], &completed));
|
||||
// Value 4.0 → keep (at threshold)
|
||||
assert!(!pruner.should_prune(6, 1, &[(1, 4.0)], &completed));
|
||||
// Value 3.0 → prune
|
||||
assert!(pruner.should_prune(6, 1, &[(1, 3.0)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reduction_factor_2() {
|
||||
let pruner = SuccessiveHalvingPruner::new()
|
||||
.reduction_factor(2)
|
||||
.min_resource(1)
|
||||
.max_resource(16)
|
||||
.direction(Direction::Minimize);
|
||||
|
||||
// Rungs: 1, 2, 4, 8, 16
|
||||
assert_eq!(pruner.rung_steps(), vec![1, 2, 4, 8, 16]);
|
||||
|
||||
// 4 completed trials at rung step=1
|
||||
let completed: Vec<_> = (0..4)
|
||||
.map(|i| make_trial(i, &[(1, (i + 1) as f64)]))
|
||||
.collect();
|
||||
|
||||
// With eta=2, 5 total. ceil(5/2) = 3 keep. Threshold = 3.0
|
||||
// Value 3.0 → keep
|
||||
assert!(!pruner.should_prune(4, 1, &[(1, 3.0)], &completed));
|
||||
// Value 4.0 → prune
|
||||
assert!(pruner.should_prune(4, 1, &[(1, 4.0)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reduction_factor_4() {
|
||||
let pruner = SuccessiveHalvingPruner::new()
|
||||
.reduction_factor(4)
|
||||
.min_resource(1)
|
||||
.max_resource(64)
|
||||
.direction(Direction::Minimize);
|
||||
|
||||
// Rungs: 1, 4, 16, 64
|
||||
assert_eq!(pruner.rung_steps(), vec![1, 4, 16, 64]);
|
||||
|
||||
// 12 completed trials at rung step=1
|
||||
let completed: Vec<_> = (0..12)
|
||||
.map(|i| make_trial(i, &[(1, (i + 1) as f64)]))
|
||||
.collect();
|
||||
|
||||
// With eta=4, 13 total. ceil(13/4) = 4 keep. Threshold = 4.0
|
||||
// Value 4.0 → keep
|
||||
assert!(!pruner.should_prune(12, 1, &[(1, 4.0)], &completed));
|
||||
// Value 5.0 → prune
|
||||
assert!(pruner.should_prune(12, 1, &[(1, 5.0)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_contiguous_steps() {
|
||||
let pruner = SuccessiveHalvingPruner::new().direction(Direction::Minimize);
|
||||
|
||||
// Trials reporting at rung step=3 (not step=1)
|
||||
let completed: Vec<_> = (0..6)
|
||||
.map(|i| make_trial(i, &[(3, (i + 1) as f64)]))
|
||||
.collect();
|
||||
|
||||
// Current trial reports at step 5 (between rung 3 and rung 9)
|
||||
// Highest rung <= 5 is 3. Use value at rung step 3.
|
||||
// Trial has value at step 3 → use it
|
||||
assert!(!pruner.should_prune(6, 5, &[(3, 2.0)], &completed));
|
||||
assert!(pruner.should_prune(6, 5, &[(3, 5.0)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_prune_at_max_resource() {
|
||||
let pruner = SuccessiveHalvingPruner::new();
|
||||
let completed: Vec<_> = (0..9)
|
||||
.map(|i| make_trial(i, &[(81, (i + 1) as f64)]))
|
||||
.collect();
|
||||
|
||||
// At the max resource rung, never prune (trial should complete)
|
||||
assert!(!pruner.should_prune(9, 81, &[(81, 100.0)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn includes_pruned_trials_in_comparison() {
|
||||
let pruner = SuccessiveHalvingPruner::new().direction(Direction::Minimize);
|
||||
|
||||
// Mix of completed and pruned trials at rung step=1
|
||||
let completed = vec![
|
||||
make_trial(0, &[(1, 1.0)]),
|
||||
make_trial(1, &[(1, 2.0)]),
|
||||
make_pruned_trial(2, &[(1, 8.0)]),
|
||||
make_pruned_trial(3, &[(1, 9.0)]),
|
||||
make_pruned_trial(4, &[(1, 10.0)]),
|
||||
];
|
||||
|
||||
// 6 total. ceil(6/3) = 2 keep. Threshold = 2.0
|
||||
// Value 2.0 → keep
|
||||
assert!(!pruner.should_prune(5, 1, &[(1, 2.0)], &completed));
|
||||
// Value 3.0 → prune
|
||||
assert!(pruner.should_prune(5, 1, &[(1, 3.0)], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "min_resource must be > 0")]
|
||||
fn rejects_zero_min_resource() {
|
||||
let _ = SuccessiveHalvingPruner::new().min_resource(0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "max_resource must be > 0")]
|
||||
fn rejects_zero_max_resource() {
|
||||
let _ = SuccessiveHalvingPruner::new().max_resource(0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "reduction_factor must be >= 2")]
|
||||
fn rejects_reduction_factor_one() {
|
||||
let _ = SuccessiveHalvingPruner::new().reduction_factor(1);
|
||||
}
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
use super::Pruner;
|
||||
use crate::sampler::CompletedTrial;
|
||||
|
||||
/// Prune trials whose intermediate values exceed fixed thresholds.
|
||||
///
|
||||
/// Useful for cutting off trials that are clearly diverging or stuck
|
||||
/// at bad values early in training.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::pruner::ThresholdPruner;
|
||||
///
|
||||
/// // Prune if the intermediate value exceeds 100.0 or falls below 0.0
|
||||
/// let pruner = ThresholdPruner::new().upper(100.0).lower(0.0);
|
||||
/// ```
|
||||
pub struct ThresholdPruner {
|
||||
/// Prune if intermediate value is greater than this. `None` = no upper bound.
|
||||
upper: Option<f64>,
|
||||
/// Prune if intermediate value is less than this. `None` = no lower bound.
|
||||
lower: Option<f64>,
|
||||
}
|
||||
|
||||
impl ThresholdPruner {
|
||||
/// Create a new `ThresholdPruner` with no thresholds set.
|
||||
///
|
||||
/// By default, no pruning occurs. Use [`upper`](Self::upper) and
|
||||
/// [`lower`](Self::lower) to set bounds.
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
upper: None,
|
||||
lower: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the upper threshold. Trials with intermediate values above this
|
||||
/// will be pruned.
|
||||
#[must_use]
|
||||
pub fn upper(mut self, threshold: f64) -> Self {
|
||||
self.upper = Some(threshold);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the lower threshold. Trials with intermediate values below this
|
||||
/// will be pruned.
|
||||
#[must_use]
|
||||
pub fn lower(mut self, threshold: f64) -> Self {
|
||||
self.lower = Some(threshold);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ThresholdPruner {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Pruner for ThresholdPruner {
|
||||
fn should_prune(
|
||||
&self,
|
||||
_trial_id: u64,
|
||||
_step: u64,
|
||||
intermediate_values: &[(u64, f64)],
|
||||
_completed_trials: &[CompletedTrial],
|
||||
) -> bool {
|
||||
let Some(&(_, latest_value)) = intermediate_values.last() else {
|
||||
return false;
|
||||
};
|
||||
if let Some(upper) = self.upper
|
||||
&& latest_value > upper
|
||||
{
|
||||
return true;
|
||||
}
|
||||
if let Some(lower) = self.lower
|
||||
&& latest_value < lower
|
||||
{
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
}
|
||||
@@ -1,531 +0,0 @@
|
||||
use core::cmp::Ordering;
|
||||
|
||||
use super::Pruner;
|
||||
use crate::sampler::CompletedTrial;
|
||||
use crate::types::{Direction, TrialState};
|
||||
|
||||
/// Prune trials using a Wilcoxon signed-rank test comparing intermediate
|
||||
/// values against the best completed trial.
|
||||
///
|
||||
/// More principled than `MedianPruner` for noisy objectives — it accounts
|
||||
/// for the paired nature of step-aligned comparisons and doesn't prune
|
||||
/// on random fluctuations.
|
||||
///
|
||||
/// The test compares intermediate values at matching steps between the
|
||||
/// current trial and the best completed trial. If the current trial is
|
||||
/// statistically significantly worse (p < threshold), it is pruned.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::Direction;
|
||||
/// use optimizer::pruner::WilcoxonPruner;
|
||||
///
|
||||
/// let pruner = WilcoxonPruner::new(Direction::Minimize)
|
||||
/// .p_value_threshold(0.05)
|
||||
/// .n_warmup_steps(5)
|
||||
/// .n_min_trials(1);
|
||||
/// ```
|
||||
pub struct WilcoxonPruner {
|
||||
/// Significance level (default 0.05). Lower = more conservative.
|
||||
p_value_threshold: f64,
|
||||
/// Don't prune in the first N steps (let the trial warm up).
|
||||
n_warmup_steps: u64,
|
||||
/// Require at least N completed trials before pruning.
|
||||
n_min_trials: usize,
|
||||
/// The optimization direction.
|
||||
direction: Direction,
|
||||
}
|
||||
|
||||
impl WilcoxonPruner {
|
||||
/// Create a new `WilcoxonPruner` for the given optimization direction.
|
||||
///
|
||||
/// By default, `p_value_threshold` is 0.05, `n_warmup_steps` is 0,
|
||||
/// and `n_min_trials` is 1.
|
||||
#[must_use]
|
||||
pub fn new(direction: Direction) -> Self {
|
||||
Self {
|
||||
p_value_threshold: 0.05,
|
||||
n_warmup_steps: 0,
|
||||
n_min_trials: 1,
|
||||
direction,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the p-value threshold for significance.
|
||||
///
|
||||
/// Must be in (0.0, 1.0). Lower values are more conservative (harder to prune).
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `p` is not in the open interval (0.0, 1.0).
|
||||
#[must_use]
|
||||
pub fn p_value_threshold(mut self, p: f64) -> Self {
|
||||
assert!(
|
||||
p > 0.0 && p < 1.0,
|
||||
"p_value_threshold must be in (0.0, 1.0)"
|
||||
);
|
||||
self.p_value_threshold = p;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the number of warmup steps. No pruning occurs before this step.
|
||||
#[must_use]
|
||||
pub fn n_warmup_steps(mut self, n: u64) -> Self {
|
||||
self.n_warmup_steps = n;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the minimum number of completed trials required before pruning.
|
||||
#[must_use]
|
||||
pub fn n_min_trials(mut self, n: usize) -> Self {
|
||||
self.n_min_trials = n;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Pruner for WilcoxonPruner {
|
||||
fn should_prune(
|
||||
&self,
|
||||
_trial_id: u64,
|
||||
step: u64,
|
||||
intermediate_values: &[(u64, f64)],
|
||||
completed_trials: &[CompletedTrial],
|
||||
) -> bool {
|
||||
if step < self.n_warmup_steps {
|
||||
return false;
|
||||
}
|
||||
|
||||
let completed: Vec<&CompletedTrial> = completed_trials
|
||||
.iter()
|
||||
.filter(|t| t.state == TrialState::Complete)
|
||||
.collect();
|
||||
|
||||
if completed.len() < self.n_min_trials {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Find the best completed trial by final objective value.
|
||||
let best = match self.direction {
|
||||
Direction::Minimize => completed
|
||||
.iter()
|
||||
.min_by(|a, b| a.value.partial_cmp(&b.value).unwrap_or(Ordering::Equal)),
|
||||
Direction::Maximize => completed
|
||||
.iter()
|
||||
.max_by(|a, b| a.value.partial_cmp(&b.value).unwrap_or(Ordering::Equal)),
|
||||
};
|
||||
|
||||
let Some(best) = best else {
|
||||
return false;
|
||||
};
|
||||
|
||||
// Pair intermediate values at matching steps.
|
||||
let pairs: Vec<(f64, f64)> = intermediate_values
|
||||
.iter()
|
||||
.filter_map(|&(s, current_v)| {
|
||||
best.intermediate_values
|
||||
.iter()
|
||||
.find(|(bs, _)| *bs == s)
|
||||
.map(|&(_, best_v)| (current_v, best_v))
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Need at least 6 pairs for a meaningful test.
|
||||
if pairs.len() < 6 {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Compute signed differences: current - best.
|
||||
// For minimization: positive diff means current is worse.
|
||||
// For maximization: negative diff means current is worse.
|
||||
let differences: Vec<f64> = pairs
|
||||
.iter()
|
||||
.map(|&(current, best_v)| current - best_v)
|
||||
.collect();
|
||||
|
||||
// Run the Wilcoxon signed-rank test.
|
||||
let p_value = wilcoxon_signed_rank_test(&differences, self.direction);
|
||||
|
||||
p_value < self.p_value_threshold
|
||||
}
|
||||
}
|
||||
|
||||
/// Perform a one-sided Wilcoxon signed-rank test.
|
||||
///
|
||||
/// Tests whether the values tend to be worse than zero (positive for
|
||||
/// minimization, negative for maximization).
|
||||
///
|
||||
/// Returns a p-value. Small p-values indicate the current trial is
|
||||
/// significantly worse.
|
||||
fn wilcoxon_signed_rank_test(differences: &[f64], direction: Direction) -> f64 {
|
||||
// 1. Remove zero differences.
|
||||
let nonzero: Vec<f64> = differences.iter().copied().filter(|d| *d != 0.0).collect();
|
||||
let n = nonzero.len();
|
||||
|
||||
if n < 6 {
|
||||
return 1.0; // Not enough data
|
||||
}
|
||||
|
||||
// 2. Rank by absolute value.
|
||||
let mut abs_ranked: Vec<(usize, f64, f64)> = nonzero
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, &d)| (i, d.abs(), d))
|
||||
.collect();
|
||||
abs_ranked.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(Ordering::Equal));
|
||||
|
||||
// 3. Assign ranks with tie correction.
|
||||
let ranks = assign_ranks(&abs_ranked);
|
||||
|
||||
// 4. Compute W+ (sum of ranks for positive differences) and
|
||||
// W- (sum of ranks for negative differences).
|
||||
let mut w_plus = 0.0;
|
||||
let mut w_minus = 0.0;
|
||||
for (i, &(_, _, orig)) in abs_ranked.iter().enumerate() {
|
||||
if orig > 0.0 {
|
||||
w_plus += ranks[i];
|
||||
} else {
|
||||
w_minus += ranks[i];
|
||||
}
|
||||
}
|
||||
|
||||
// For one-sided test:
|
||||
// - Minimization: we want to detect positive diffs (current worse).
|
||||
// Large W+ means significantly worse. Test statistic = W-.
|
||||
// - Maximization: we want to detect negative diffs (current worse).
|
||||
// Large W- means significantly worse. Test statistic = W+.
|
||||
let w = match direction {
|
||||
Direction::Minimize => w_minus,
|
||||
Direction::Maximize => w_plus,
|
||||
};
|
||||
|
||||
// 5. Normal approximation for the p-value.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
let n_f = n as f64;
|
||||
let mean = n_f * (n_f + 1.0) / 4.0;
|
||||
let variance = n_f * (n_f + 1.0) * (2.0 * n_f + 1.0) / 24.0;
|
||||
|
||||
// Tie correction for variance.
|
||||
let tie_correction = compute_tie_correction(&ranks);
|
||||
let adjusted_variance = variance - tie_correction;
|
||||
|
||||
if adjusted_variance <= 0.0 {
|
||||
return 1.0;
|
||||
}
|
||||
|
||||
let std_dev = adjusted_variance.sqrt();
|
||||
|
||||
// Continuity correction: shift W by 0.5 towards mean.
|
||||
let continuity = if w < mean { 0.5 } else { -0.5 };
|
||||
let z = (w + continuity - mean) / std_dev;
|
||||
|
||||
// One-sided p-value (lower tail): probability that the test statistic
|
||||
// is this small or smaller under H0.
|
||||
normal_cdf(z)
|
||||
}
|
||||
|
||||
/// Assign average ranks, handling ties.
|
||||
fn assign_ranks(sorted: &[(usize, f64, f64)]) -> Vec<f64> {
|
||||
let n = sorted.len();
|
||||
let mut ranks = vec![0.0; n];
|
||||
let mut i = 0;
|
||||
|
||||
while i < n {
|
||||
let mut j = i;
|
||||
// Find all items tied with sorted[i].
|
||||
while j < n
|
||||
&& (sorted[j].1 - sorted[i].1).abs() < f64::EPSILON * sorted[i].1.max(1.0) * 100.0
|
||||
{
|
||||
j += 1;
|
||||
}
|
||||
// Average rank for the tie group. Ranks are 1-based.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
let avg_rank = (i + 1 + j) as f64 / 2.0;
|
||||
for rank in ranks.iter_mut().take(j).skip(i) {
|
||||
*rank = avg_rank;
|
||||
}
|
||||
i = j;
|
||||
}
|
||||
|
||||
ranks
|
||||
}
|
||||
|
||||
/// Compute the tie correction term for the variance.
|
||||
/// For each tie group of size t, subtract t^3 - t from the sum,
|
||||
/// then divide by 48.
|
||||
fn compute_tie_correction(ranks: &[f64]) -> f64 {
|
||||
let mut correction = 0.0;
|
||||
let mut i = 0;
|
||||
|
||||
while i < ranks.len() {
|
||||
let mut j = i;
|
||||
while j < ranks.len() && (ranks[j] - ranks[i]).abs() < f64::EPSILON {
|
||||
j += 1;
|
||||
}
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
let t = (j - i) as f64;
|
||||
if t > 1.0 {
|
||||
correction += t * t * t - t;
|
||||
}
|
||||
i = j;
|
||||
}
|
||||
|
||||
correction / 48.0
|
||||
}
|
||||
|
||||
/// Standard normal CDF using an approximation (Abramowitz & Stegun).
|
||||
fn normal_cdf(x: f64) -> f64 {
|
||||
// Use the complementary error function relationship:
|
||||
// Φ(x) = 0.5 * erfc(-x / √2)
|
||||
0.5 * erfc(-x / core::f64::consts::SQRT_2)
|
||||
}
|
||||
|
||||
/// Complementary error function approximation.
|
||||
/// Maximum error: 1.5 × 10⁻⁷ (Abramowitz & Stegun formula 7.1.26).
|
||||
fn erfc(x: f64) -> f64 {
|
||||
let t = 1.0 / (1.0 + 0.327_591_1 * x.abs());
|
||||
let poly = t
|
||||
* (0.254_829_592
|
||||
+ t * (-0.284_496_736
|
||||
+ t * (1.421_413_741 + t * (-1.453_152_027 + t * 1.061_405_429))));
|
||||
let result = poly * (-x * x).exp();
|
||||
|
||||
if x >= 0.0 { result } else { 2.0 - result }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn trial_with_values(
|
||||
id: u64,
|
||||
value: f64,
|
||||
intermediate_values: Vec<(u64, f64)>,
|
||||
) -> CompletedTrial {
|
||||
CompletedTrial::with_intermediate_values(
|
||||
id,
|
||||
HashMap::new(),
|
||||
HashMap::new(),
|
||||
HashMap::new(),
|
||||
value,
|
||||
intermediate_values,
|
||||
HashMap::new(),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_prune_during_warmup() {
|
||||
let pruner = WilcoxonPruner::new(Direction::Minimize).n_warmup_steps(10);
|
||||
let completed = vec![trial_with_values(
|
||||
0,
|
||||
0.1,
|
||||
(0..20).map(|s| (s, 0.1)).collect(),
|
||||
)];
|
||||
let current: Vec<(u64, f64)> = (0..8).map(|s| (s, 100.0)).collect();
|
||||
assert!(!pruner.should_prune(1, 7, ¤t, &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_prune_with_insufficient_trials() {
|
||||
let pruner = WilcoxonPruner::new(Direction::Minimize).n_min_trials(5);
|
||||
let completed = vec![trial_with_values(
|
||||
0,
|
||||
0.1,
|
||||
(0..20).map(|s| (s, 0.1)).collect(),
|
||||
)];
|
||||
let current: Vec<(u64, f64)> = (0..10).map(|s| (s, 100.0)).collect();
|
||||
assert!(!pruner.should_prune(1, 9, ¤t, &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_prune_with_fewer_than_6_pairs() {
|
||||
let pruner = WilcoxonPruner::new(Direction::Minimize);
|
||||
let completed = vec![trial_with_values(
|
||||
0,
|
||||
0.1,
|
||||
(0..5).map(|s| (s, 0.1)).collect(),
|
||||
)];
|
||||
// Only 5 matching steps
|
||||
let current: Vec<(u64, f64)> = (0..5).map(|s| (s, 100.0)).collect();
|
||||
assert!(!pruner.should_prune(1, 4, ¤t, &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prune_when_consistently_worse_minimize() {
|
||||
let pruner = WilcoxonPruner::new(Direction::Minimize);
|
||||
|
||||
// Best trial has low values.
|
||||
let best_values: Vec<(u64, f64)> = (0..20).map(|s| (s, 0.1)).collect();
|
||||
let completed = vec![trial_with_values(0, 0.1, best_values)];
|
||||
|
||||
// Current trial is consistently much worse.
|
||||
let current: Vec<(u64, f64)> = (0..20).map(|s| (s, 10.0)).collect();
|
||||
assert!(pruner.should_prune(1, 19, ¤t, &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prune_when_consistently_worse_maximize() {
|
||||
let pruner = WilcoxonPruner::new(Direction::Maximize);
|
||||
|
||||
// Best trial has high values.
|
||||
let best_values: Vec<(u64, f64)> = (0..20).map(|s| (s, 10.0)).collect();
|
||||
let completed = vec![trial_with_values(0, 10.0, best_values)];
|
||||
|
||||
// Current trial is consistently much worse.
|
||||
let current: Vec<(u64, f64)> = (0..20).map(|s| (s, 0.1)).collect();
|
||||
assert!(pruner.should_prune(1, 19, ¤t, &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_prune_when_statistically_similar() {
|
||||
let pruner = WilcoxonPruner::new(Direction::Minimize);
|
||||
|
||||
// Best trial and current trial have very similar values with noise.
|
||||
let best_values: Vec<(u64, f64)> = (0..20_u64)
|
||||
.map(|s| {
|
||||
let noise = if s.is_multiple_of(2) { 0.01 } else { -0.01 };
|
||||
(s, 1.0 + noise)
|
||||
})
|
||||
.collect();
|
||||
let completed = vec![trial_with_values(0, 1.0, best_values)];
|
||||
|
||||
// Current trial is similar — alternating above/below.
|
||||
let current: Vec<(u64, f64)> = (0..20_u64)
|
||||
.map(|s| {
|
||||
let noise = if s.is_multiple_of(2) { -0.01 } else { 0.01 };
|
||||
(s, 1.0 + noise)
|
||||
})
|
||||
.collect();
|
||||
assert!(!pruner.should_prune(1, 19, ¤t, &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selects_best_trial_minimize() {
|
||||
let pruner = WilcoxonPruner::new(Direction::Minimize);
|
||||
|
||||
// Two completed trials: trial 0 is better (lower).
|
||||
let completed = vec![
|
||||
trial_with_values(0, 0.1, (0..20).map(|s| (s, 0.1)).collect()),
|
||||
trial_with_values(1, 5.0, (0..20).map(|s| (s, 5.0)).collect()),
|
||||
];
|
||||
|
||||
// Current trial is worse than the best but similar to the second.
|
||||
let current: Vec<(u64, f64)> = (0..20).map(|s| (s, 5.0)).collect();
|
||||
assert!(pruner.should_prune(2, 19, ¤t, &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selects_best_trial_maximize() {
|
||||
let pruner = WilcoxonPruner::new(Direction::Maximize);
|
||||
|
||||
// Two completed trials: trial 1 is better (higher).
|
||||
let completed = vec![
|
||||
trial_with_values(0, 0.1, (0..20).map(|s| (s, 0.1)).collect()),
|
||||
trial_with_values(1, 10.0, (0..20).map(|s| (s, 10.0)).collect()),
|
||||
];
|
||||
|
||||
// Current trial is worse than the best.
|
||||
let current: Vec<(u64, f64)> = (0..20).map(|s| (s, 0.1)).collect();
|
||||
assert!(pruner.should_prune(2, 19, ¤t, &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ignores_pruned_trials() {
|
||||
let pruner = WilcoxonPruner::new(Direction::Minimize);
|
||||
|
||||
// Only a pruned trial — no complete trials.
|
||||
let mut trial = trial_with_values(0, 0.1, (0..20).map(|s| (s, 0.1)).collect());
|
||||
trial.state = TrialState::Pruned;
|
||||
let completed = vec![trial];
|
||||
|
||||
let current: Vec<(u64, f64)> = (0..20).map(|s| (s, 100.0)).collect();
|
||||
assert!(!pruner.should_prune(1, 19, ¤t, &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lower_p_value_is_more_conservative() {
|
||||
let strict = WilcoxonPruner::new(Direction::Minimize).p_value_threshold(0.001);
|
||||
let lenient = WilcoxonPruner::new(Direction::Minimize).p_value_threshold(0.1);
|
||||
|
||||
let completed = vec![trial_with_values(
|
||||
0,
|
||||
0.1,
|
||||
(0..20).map(|s| (s, 0.1)).collect(),
|
||||
)];
|
||||
|
||||
// Moderately worse — should pass lenient but maybe not strict.
|
||||
let current: Vec<(u64, f64)> = (0..20)
|
||||
.map(|s| if s < 15 { (s, 0.2) } else { (s, 0.15) })
|
||||
.collect();
|
||||
|
||||
let lenient_prunes = lenient.should_prune(1, 19, ¤t, &completed);
|
||||
let strict_prunes = strict.should_prune(1, 19, ¤t, &completed);
|
||||
|
||||
// A stricter threshold should never prune when a lenient one doesn't.
|
||||
if !lenient_prunes {
|
||||
assert!(!strict_prunes);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "p_value_threshold must be in (0.0, 1.0)")]
|
||||
fn panics_on_zero_p_value() {
|
||||
let _ = WilcoxonPruner::new(Direction::Minimize).p_value_threshold(0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "p_value_threshold must be in (0.0, 1.0)")]
|
||||
fn panics_on_one_p_value() {
|
||||
let _ = WilcoxonPruner::new(Direction::Minimize).p_value_threshold(1.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn correct_signed_rank_statistic() {
|
||||
// Known example: differences [1, 2, 3, 4, 5, 6] (all positive).
|
||||
// Ranks: 1, 2, 3, 4, 5, 6. W+ = 21, W- = 0.
|
||||
// For minimization (testing if positive = worse), W- = 0.
|
||||
// This should give a very small p-value.
|
||||
let diffs = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0];
|
||||
let p = wilcoxon_signed_rank_test(&diffs, Direction::Minimize);
|
||||
assert!(
|
||||
p < 0.05,
|
||||
"p-value {p} should be < 0.05 for all-positive diffs"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn symmetric_differences_not_significant() {
|
||||
// Balanced differences: half positive, half negative.
|
||||
let diffs = vec![1.0, -1.0, 2.0, -2.0, 3.0, -3.0, 4.0, -4.0];
|
||||
let p = wilcoxon_signed_rank_test(&diffs, Direction::Minimize);
|
||||
assert!(p > 0.05, "p-value {p} should be > 0.05 for symmetric diffs");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normal_cdf_known_values() {
|
||||
assert!((normal_cdf(0.0) - 0.5).abs() < 1e-6);
|
||||
assert!(normal_cdf(-10.0) < 1e-6);
|
||||
assert!((normal_cdf(10.0) - 1.0).abs() < 1e-6);
|
||||
assert!((normal_cdf(-1.96) - 0.025).abs() < 0.001);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_intermediate_values() {
|
||||
let pruner = WilcoxonPruner::new(Direction::Minimize);
|
||||
let completed = vec![trial_with_values(
|
||||
0,
|
||||
0.1,
|
||||
(0..20).map(|s| (s, 0.1)).collect(),
|
||||
)];
|
||||
assert!(!pruner.should_prune(1, 0, &[], &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_completed_trials() {
|
||||
let pruner = WilcoxonPruner::new(Direction::Minimize);
|
||||
let current: Vec<(u64, f64)> = (0..20).map(|s| (s, 1.0)).collect();
|
||||
assert!(!pruner.should_prune(1, 19, ¤t, &[]));
|
||||
}
|
||||
}
|
||||
@@ -1,691 +0,0 @@
|
||||
//! BOHB (Bayesian Optimization + `HyperBand`) sampler.
|
||||
//!
|
||||
//! BOHB combines TPE's model-guided sampling with Hyperband's budget-aware
|
||||
//! evaluation. Instead of building one global TPE model, BOHB conditions
|
||||
//! its TPE model on trials evaluated at a specific budget level, giving
|
||||
//! better-calibrated proposals for each rung of the Hyperband schedule.
|
||||
//!
|
||||
//! # How it works
|
||||
//!
|
||||
//! 1. Compute all Hyperband rung steps (budget levels) from the config.
|
||||
//! 2. On each `sample()` call, scan the history's `intermediate_values`
|
||||
//! to find the **largest budget level** with enough observations
|
||||
//! (`>= min_points_in_model`).
|
||||
//! 3. Build a filtered history where each trial's `value` is replaced
|
||||
//! with its intermediate value at that budget level.
|
||||
//! 4. Delegate to an internal [`TpeSampler`] for the actual sampling.
|
||||
//! 5. Fall back to random sampling if no budget level has enough data.
|
||||
//!
|
||||
//! # Examples
|
||||
//!
|
||||
//! ```
|
||||
//! use optimizer::sampler::bohb::BohbSampler;
|
||||
//! use optimizer::{Direction, Study};
|
||||
//!
|
||||
//! let bohb = BohbSampler::new();
|
||||
//! let pruner = bohb.matching_pruner(Direction::Minimize);
|
||||
//! let study: Study<f64> = Study::with_sampler_and_pruner(Direction::Minimize, bohb, pruner);
|
||||
//! ```
|
||||
//!
|
||||
//! Using the builder for custom configuration:
|
||||
//!
|
||||
//! ```
|
||||
//! use optimizer::sampler::bohb::BohbSampler;
|
||||
//!
|
||||
//! let bohb = BohbSampler::builder()
|
||||
//! .min_resource(1)
|
||||
//! .max_resource(81)
|
||||
//! .reduction_factor(3)
|
||||
//! .min_points_in_model(10)
|
||||
//! .seed(42)
|
||||
//! .build()
|
||||
//! .unwrap();
|
||||
//! ```
|
||||
|
||||
use crate::distribution::Distribution;
|
||||
use crate::error::Result;
|
||||
use crate::param::ParamValue;
|
||||
use crate::pruner::HyperbandPruner;
|
||||
use crate::sampler::tpe::TpeSampler;
|
||||
use crate::sampler::{CompletedTrial, Sampler};
|
||||
use crate::types::Direction;
|
||||
|
||||
/// A BOHB sampler that combines TPE with Hyperband budget awareness.
|
||||
///
|
||||
/// BOHB filters trial history by budget level before delegating to TPE,
|
||||
/// so the surrogate model is conditioned on trials evaluated at the same
|
||||
/// resource level. This produces better-calibrated parameter proposals
|
||||
/// than using a single global model across all budgets.
|
||||
///
|
||||
/// Use [`BohbSampler::matching_pruner`] to create a [`HyperbandPruner`]
|
||||
/// with matching parameters.
|
||||
pub struct BohbSampler {
|
||||
min_resource: u64,
|
||||
max_resource: u64,
|
||||
reduction_factor: u64,
|
||||
min_points_in_model: usize,
|
||||
tpe: TpeSampler,
|
||||
}
|
||||
|
||||
impl BohbSampler {
|
||||
/// Creates a new BOHB sampler with default settings.
|
||||
///
|
||||
/// Defaults:
|
||||
/// - `min_resource`: 1
|
||||
/// - `max_resource`: 81
|
||||
/// - `reduction_factor`: 3
|
||||
/// - `min_points_in_model`: 10
|
||||
/// - TPE: default settings
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
min_resource: 1,
|
||||
max_resource: 81,
|
||||
reduction_factor: 3,
|
||||
min_points_in_model: 10,
|
||||
tpe: TpeSampler::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a builder for configuring a BOHB sampler.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::sampler::bohb::BohbSampler;
|
||||
///
|
||||
/// let sampler = BohbSampler::builder()
|
||||
/// .min_resource(1)
|
||||
/// .max_resource(27)
|
||||
/// .reduction_factor(3)
|
||||
/// .min_points_in_model(5)
|
||||
/// .seed(42)
|
||||
/// .build()
|
||||
/// .unwrap();
|
||||
/// ```
|
||||
#[must_use]
|
||||
pub fn builder() -> BohbSamplerBuilder {
|
||||
BohbSamplerBuilder::new()
|
||||
}
|
||||
|
||||
/// Creates a [`HyperbandPruner`] with matching Hyperband parameters.
|
||||
///
|
||||
/// This ensures the pruner's budget schedule is consistent with the
|
||||
/// budget levels used by BOHB for model conditioning.
|
||||
#[must_use]
|
||||
pub fn matching_pruner(&self, direction: Direction) -> HyperbandPruner {
|
||||
HyperbandPruner::new()
|
||||
.min_resource(self.min_resource)
|
||||
.max_resource(self.max_resource)
|
||||
.reduction_factor(self.reduction_factor)
|
||||
.direction(direction)
|
||||
}
|
||||
|
||||
/// Compute all unique budget levels (rung steps) across all Hyperband brackets.
|
||||
///
|
||||
/// Returns sorted ascending.
|
||||
#[allow(
|
||||
clippy::cast_precision_loss,
|
||||
clippy::cast_possible_truncation,
|
||||
clippy::cast_sign_loss
|
||||
)]
|
||||
fn all_budget_levels(&self) -> Vec<u64> {
|
||||
let eta = self.reduction_factor as f64;
|
||||
let ratio = self.max_resource as f64 / self.min_resource as f64;
|
||||
let s_max = (ratio.ln() / eta.ln()).floor() as u64;
|
||||
|
||||
let mut levels = Vec::new();
|
||||
for bracket in 0..=s_max {
|
||||
let exponent = s_max.saturating_sub(bracket);
|
||||
let min_resource_bracket =
|
||||
(self.max_resource as f64 / eta.powi(exponent as i32)).ceil() as u64;
|
||||
|
||||
let mut rung: u32 = 0;
|
||||
while let Some(power) = self.reduction_factor.checked_pow(rung) {
|
||||
let step = min_resource_bracket.saturating_mul(power);
|
||||
if step > self.max_resource {
|
||||
break;
|
||||
}
|
||||
levels.push(step);
|
||||
rung += 1;
|
||||
}
|
||||
}
|
||||
|
||||
levels.sort_unstable();
|
||||
levels.dedup();
|
||||
levels
|
||||
}
|
||||
|
||||
/// Build a filtered history for a specific budget level.
|
||||
///
|
||||
/// For each trial that has an intermediate value at the given budget step,
|
||||
/// creates a new `CompletedTrial` with `value` replaced by the intermediate
|
||||
/// value at that step.
|
||||
fn filter_history_for_budget(history: &[CompletedTrial], budget: u64) -> Vec<CompletedTrial> {
|
||||
history
|
||||
.iter()
|
||||
.filter_map(|trial| {
|
||||
trial
|
||||
.intermediate_values
|
||||
.iter()
|
||||
.find(|(step, _)| *step == budget)
|
||||
.map(|(_, iv)| CompletedTrial {
|
||||
id: trial.id,
|
||||
params: trial.params.clone(),
|
||||
distributions: trial.distributions.clone(),
|
||||
param_labels: trial.param_labels.clone(),
|
||||
value: *iv,
|
||||
intermediate_values: trial.intermediate_values.clone(),
|
||||
state: trial.state,
|
||||
user_attrs: trial.user_attrs.clone(),
|
||||
constraints: trial.constraints.clone(),
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for BohbSampler {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Sampler for BohbSampler {
|
||||
fn sample(
|
||||
&self,
|
||||
distribution: &Distribution,
|
||||
trial_id: u64,
|
||||
history: &[CompletedTrial],
|
||||
) -> ParamValue {
|
||||
// Find the largest budget level with enough observations
|
||||
let levels = self.all_budget_levels();
|
||||
|
||||
for &budget in levels.iter().rev() {
|
||||
let count = history
|
||||
.iter()
|
||||
.filter(|t| {
|
||||
t.intermediate_values
|
||||
.iter()
|
||||
.any(|(step, _)| *step == budget)
|
||||
})
|
||||
.count();
|
||||
|
||||
if count >= self.min_points_in_model {
|
||||
let filtered = Self::filter_history_for_budget(history, budget);
|
||||
return self.tpe.sample(distribution, trial_id, &filtered);
|
||||
}
|
||||
}
|
||||
|
||||
// Not enough data at any budget level: delegate to TPE with empty history
|
||||
// which triggers its uniform-random startup behavior.
|
||||
self.tpe.sample(distribution, trial_id, &[])
|
||||
}
|
||||
}
|
||||
|
||||
/// Builder for configuring a [`BohbSampler`].
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::sampler::bohb::BohbSamplerBuilder;
|
||||
///
|
||||
/// let sampler = BohbSamplerBuilder::new()
|
||||
/// .min_resource(1)
|
||||
/// .max_resource(81)
|
||||
/// .reduction_factor(3)
|
||||
/// .gamma(0.15)
|
||||
/// .seed(42)
|
||||
/// .build()
|
||||
/// .unwrap();
|
||||
/// ```
|
||||
pub struct BohbSamplerBuilder {
|
||||
min_resource: u64,
|
||||
max_resource: u64,
|
||||
reduction_factor: u64,
|
||||
min_points_in_model: usize,
|
||||
tpe_builder: crate::sampler::tpe::TpeSamplerBuilder,
|
||||
}
|
||||
|
||||
impl BohbSamplerBuilder {
|
||||
/// Creates a new builder with default settings.
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
min_resource: 1,
|
||||
max_resource: 81,
|
||||
reduction_factor: 3,
|
||||
min_points_in_model: 10,
|
||||
tpe_builder: crate::sampler::tpe::TpeSamplerBuilder::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets the minimum resource (budget) per trial.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `r` is 0.
|
||||
#[must_use]
|
||||
pub fn min_resource(mut self, r: u64) -> Self {
|
||||
assert!(r > 0, "min_resource must be > 0, got {r}");
|
||||
self.min_resource = r;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the maximum resource (budget) per trial.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `r` is 0.
|
||||
#[must_use]
|
||||
pub fn max_resource(mut self, r: u64) -> Self {
|
||||
assert!(r > 0, "max_resource must be > 0, got {r}");
|
||||
self.max_resource = r;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the reduction factor (eta).
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if `eta` is less than 2.
|
||||
#[must_use]
|
||||
pub fn reduction_factor(mut self, eta: u64) -> Self {
|
||||
assert!(eta >= 2, "reduction_factor must be >= 2, got {eta}");
|
||||
self.reduction_factor = eta;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the minimum number of observations at a budget level before
|
||||
/// BOHB uses TPE instead of random sampling.
|
||||
#[must_use]
|
||||
pub fn min_points_in_model(mut self, n: usize) -> Self {
|
||||
self.min_points_in_model = n;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets a fixed gamma value for the internal TPE sampler.
|
||||
#[must_use]
|
||||
pub fn gamma(mut self, gamma: f64) -> Self {
|
||||
self.tpe_builder = self.tpe_builder.gamma(gamma);
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets a custom gamma strategy for the internal TPE sampler.
|
||||
#[must_use]
|
||||
pub fn gamma_strategy<G: crate::sampler::tpe::GammaStrategy + 'static>(
|
||||
mut self,
|
||||
strategy: G,
|
||||
) -> Self {
|
||||
self.tpe_builder = self.tpe_builder.gamma_strategy(strategy);
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the number of EI candidates for the internal TPE sampler.
|
||||
#[must_use]
|
||||
pub fn n_ei_candidates(mut self, n: usize) -> Self {
|
||||
self.tpe_builder = self.tpe_builder.n_ei_candidates(n);
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets a fixed KDE bandwidth for the internal TPE sampler.
|
||||
#[must_use]
|
||||
pub fn kde_bandwidth(mut self, bandwidth: f64) -> Self {
|
||||
self.tpe_builder = self.tpe_builder.kde_bandwidth(bandwidth);
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets a seed for reproducible sampling.
|
||||
#[must_use]
|
||||
pub fn seed(mut self, seed: u64) -> Self {
|
||||
self.tpe_builder = self.tpe_builder.seed(seed);
|
||||
self
|
||||
}
|
||||
|
||||
/// Builds the configured [`BohbSampler`].
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if the TPE configuration is invalid (e.g. gamma
|
||||
/// not in (0, 1) or bandwidth not positive).
|
||||
pub fn build(self) -> Result<BohbSampler> {
|
||||
let tpe = self.tpe_builder.build()?;
|
||||
Ok(BohbSampler {
|
||||
min_resource: self.min_resource,
|
||||
max_resource: self.max_resource,
|
||||
reduction_factor: self.reduction_factor,
|
||||
min_points_in_model: self.min_points_in_model,
|
||||
tpe,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for BohbSamplerBuilder {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
|
||||
use super::*;
|
||||
use crate::distribution::{FloatDistribution, IntDistribution};
|
||||
use crate::parameter::ParamId;
|
||||
use crate::types::TrialState;
|
||||
|
||||
fn make_trial_with_intermediates(
|
||||
id: u64,
|
||||
value: f64,
|
||||
params: Vec<(ParamId, ParamValue, Distribution)>,
|
||||
intermediate_values: Vec<(u64, f64)>,
|
||||
) -> CompletedTrial {
|
||||
let mut param_map = HashMap::new();
|
||||
let mut dist_map = HashMap::new();
|
||||
for (param_id, pv, dist) in params {
|
||||
param_map.insert(param_id, pv);
|
||||
dist_map.insert(param_id, dist);
|
||||
}
|
||||
CompletedTrial {
|
||||
id,
|
||||
params: param_map,
|
||||
distributions: dist_map,
|
||||
param_labels: HashMap::new(),
|
||||
value,
|
||||
intermediate_values,
|
||||
state: TrialState::Complete,
|
||||
user_attrs: HashMap::new(),
|
||||
constraints: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn budget_levels_default() {
|
||||
let bohb = BohbSampler::new();
|
||||
let levels = bohb.all_budget_levels();
|
||||
// With min=1, max=81, eta=3:
|
||||
// bracket 0: 1, 3, 9, 27, 81
|
||||
// bracket 1: 3, 9, 27, 81
|
||||
// bracket 2: 9, 27, 81
|
||||
// bracket 3: 27, 81
|
||||
// bracket 4: 81
|
||||
// Unique sorted: [1, 3, 9, 27, 81]
|
||||
assert_eq!(levels, vec![1, 3, 9, 27, 81]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn budget_levels_eta2() {
|
||||
let bohb = BohbSampler::builder()
|
||||
.min_resource(1)
|
||||
.max_resource(16)
|
||||
.reduction_factor(2)
|
||||
.build()
|
||||
.unwrap();
|
||||
let levels = bohb.all_budget_levels();
|
||||
// s_max = floor(ln(16)/ln(2)) = 4
|
||||
// bracket 0: 1, 2, 4, 8, 16
|
||||
// bracket 1: 2, 4, 8, 16
|
||||
// bracket 2: 4, 8, 16
|
||||
// bracket 3: 8, 16
|
||||
// bracket 4: 16
|
||||
// Unique sorted: [1, 2, 4, 8, 16]
|
||||
assert_eq!(levels, vec![1, 2, 4, 8, 16]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filter_history_selects_correct_budget() {
|
||||
let x_id = ParamId::new();
|
||||
let dist = Distribution::Float(FloatDistribution {
|
||||
low: 0.0,
|
||||
high: 1.0,
|
||||
log_scale: false,
|
||||
step: None,
|
||||
});
|
||||
|
||||
let history = vec![
|
||||
make_trial_with_intermediates(
|
||||
0,
|
||||
0.5,
|
||||
vec![(x_id, ParamValue::Float(0.3), dist.clone())],
|
||||
vec![(1, 0.9), (3, 0.7), (9, 0.5)],
|
||||
),
|
||||
make_trial_with_intermediates(
|
||||
1,
|
||||
0.4,
|
||||
vec![(x_id, ParamValue::Float(0.6), dist.clone())],
|
||||
vec![(1, 0.8), (3, 0.4)],
|
||||
),
|
||||
make_trial_with_intermediates(
|
||||
2,
|
||||
0.3,
|
||||
vec![(x_id, ParamValue::Float(0.1), dist.clone())],
|
||||
vec![(1, 0.7)],
|
||||
),
|
||||
];
|
||||
|
||||
// Budget 3: trials 0 and 1 have intermediate values at step 3
|
||||
let filtered = BohbSampler::filter_history_for_budget(&history, 3);
|
||||
assert_eq!(filtered.len(), 2);
|
||||
assert!((filtered[0].value - 0.7).abs() < f64::EPSILON);
|
||||
assert!((filtered[1].value - 0.4).abs() < f64::EPSILON);
|
||||
|
||||
// Budget 9: only trial 0
|
||||
let filtered = BohbSampler::filter_history_for_budget(&history, 9);
|
||||
assert_eq!(filtered.len(), 1);
|
||||
assert!((filtered[0].value - 0.5).abs() < f64::EPSILON);
|
||||
|
||||
// Budget 27: nobody
|
||||
let filtered = BohbSampler::filter_history_for_budget(&history, 27);
|
||||
assert!(filtered.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matching_pruner_has_same_params() {
|
||||
let bohb = BohbSampler::builder()
|
||||
.min_resource(2)
|
||||
.max_resource(64)
|
||||
.reduction_factor(4)
|
||||
.build()
|
||||
.unwrap();
|
||||
let pruner = bohb.matching_pruner(Direction::Minimize);
|
||||
|
||||
// We can't directly inspect HyperbandPruner fields, but we can
|
||||
// verify it was created without panicking with the same params.
|
||||
// The pruner's rung steps should match BOHB's budget levels.
|
||||
// Just verify it doesn't panic.
|
||||
drop(pruner);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fallback_to_random_when_insufficient_data() {
|
||||
let bohb = BohbSampler::builder()
|
||||
.min_points_in_model(10)
|
||||
.seed(42)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let dist = Distribution::Float(FloatDistribution {
|
||||
low: 0.0,
|
||||
high: 1.0,
|
||||
log_scale: false,
|
||||
step: None,
|
||||
});
|
||||
|
||||
// Only 3 trials with intermediate values (< min_points_in_model=10)
|
||||
let x_id = ParamId::new();
|
||||
let history: Vec<CompletedTrial> = (0..3)
|
||||
.map(|i| {
|
||||
make_trial_with_intermediates(
|
||||
i,
|
||||
i as f64,
|
||||
vec![(x_id, ParamValue::Float(i as f64 / 3.0), dist.clone())],
|
||||
vec![(1, i as f64)],
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Should not panic, should sample within bounds
|
||||
for trial_id in 0..20 {
|
||||
let val = bohb.sample(&dist, trial_id, &history);
|
||||
if let ParamValue::Float(v) = val {
|
||||
assert!((0.0..=1.0).contains(&v));
|
||||
} else {
|
||||
panic!("Expected Float");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uses_budget_level_when_enough_data() {
|
||||
let bohb = BohbSampler::builder()
|
||||
.min_points_in_model(5)
|
||||
.seed(42)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let dist = Distribution::Float(FloatDistribution {
|
||||
low: 0.0,
|
||||
high: 10.0,
|
||||
log_scale: false,
|
||||
step: None,
|
||||
});
|
||||
|
||||
// Create 20 trials with intermediate values at budget 1.
|
||||
// Good trials have x near 2.0, bad trials have x far from 2.0.
|
||||
let x_id = ParamId::new();
|
||||
let history: Vec<CompletedTrial> = (0..20)
|
||||
.map(|i| {
|
||||
let x = i as f64 / 2.0;
|
||||
let iv_at_1 = (x - 2.0).powi(2);
|
||||
make_trial_with_intermediates(
|
||||
i,
|
||||
iv_at_1, // final value same as intermediate for simplicity
|
||||
vec![(x_id, ParamValue::Float(x), dist.clone())],
|
||||
vec![(1, iv_at_1)],
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Should use TPE on filtered history at budget 1
|
||||
let val = bohb.sample(&dist, 100, &history);
|
||||
if let ParamValue::Float(v) = val {
|
||||
assert!((0.0..=10.0).contains(&v), "Value {v} out of bounds");
|
||||
} else {
|
||||
panic!("Expected Float");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefers_largest_budget_level() {
|
||||
let bohb = BohbSampler::builder()
|
||||
.min_resource(1)
|
||||
.max_resource(9)
|
||||
.reduction_factor(3)
|
||||
.min_points_in_model(3)
|
||||
.seed(42)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let dist = Distribution::Float(FloatDistribution {
|
||||
low: 0.0,
|
||||
high: 10.0,
|
||||
log_scale: false,
|
||||
step: None,
|
||||
});
|
||||
|
||||
// Budget levels: [1, 3, 9]
|
||||
assert_eq!(bohb.all_budget_levels(), vec![1, 3, 9]);
|
||||
|
||||
// Create 5 trials with intermediates at budget 1 and 3
|
||||
let x_id = ParamId::new();
|
||||
let history: Vec<CompletedTrial> = (0..5)
|
||||
.map(|i| {
|
||||
let x = i as f64;
|
||||
make_trial_with_intermediates(
|
||||
i,
|
||||
x,
|
||||
vec![(x_id, ParamValue::Float(x), dist.clone())],
|
||||
vec![(1, x * 2.0), (3, x)],
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Budget 3 has 5 observations (>= 3), budget 9 has 0.
|
||||
// BOHB should pick budget 3 (largest with enough data).
|
||||
// The filtered history at budget 3 has values [0, 1, 2, 3, 4].
|
||||
let filtered_3 = BohbSampler::filter_history_for_budget(&history, 3);
|
||||
assert_eq!(filtered_3.len(), 5);
|
||||
let filtered_9 = BohbSampler::filter_history_for_budget(&history, 9);
|
||||
assert_eq!(filtered_9.len(), 0);
|
||||
|
||||
// Should sample successfully
|
||||
let val = bohb.sample(&dist, 100, &history);
|
||||
assert!(matches!(val, ParamValue::Float(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builder_validates_tpe_params() {
|
||||
// Invalid gamma
|
||||
let result = BohbSampler::builder().gamma(1.5).build();
|
||||
assert!(result.is_err());
|
||||
|
||||
// Invalid bandwidth
|
||||
let result = BohbSampler::builder().kde_bandwidth(-1.0).build();
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "min_resource must be > 0")]
|
||||
fn builder_rejects_zero_min_resource() {
|
||||
let _ = BohbSampler::builder().min_resource(0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "max_resource must be > 0")]
|
||||
fn builder_rejects_zero_max_resource() {
|
||||
let _ = BohbSampler::builder().max_resource(0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "reduction_factor must be >= 2")]
|
||||
fn builder_rejects_small_reduction_factor() {
|
||||
let _ = BohbSampler::builder().reduction_factor(1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn int_distribution_works() {
|
||||
let bohb = BohbSampler::builder()
|
||||
.min_points_in_model(3)
|
||||
.seed(42)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let dist = Distribution::Int(IntDistribution {
|
||||
low: 0,
|
||||
high: 100,
|
||||
log_scale: false,
|
||||
step: None,
|
||||
});
|
||||
|
||||
let x_id = ParamId::new();
|
||||
let history: Vec<CompletedTrial> = (0..10)
|
||||
.map(|i| {
|
||||
make_trial_with_intermediates(
|
||||
i,
|
||||
i as f64,
|
||||
vec![(x_id, ParamValue::Int(i.cast_signed() * 10), dist.clone())],
|
||||
vec![(1, i as f64)],
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let val = bohb.sample(&dist, 100, &history);
|
||||
if let ParamValue::Int(v) = val {
|
||||
assert!((0..=100).contains(&v));
|
||||
} else {
|
||||
panic!("Expected Int");
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
-1157
File diff suppressed because it is too large
Load Diff
+32
-155
@@ -1,23 +1,15 @@
|
||||
//! Sampler trait and implementations for parameter sampling.
|
||||
|
||||
pub mod bohb;
|
||||
#[cfg(feature = "cma-es")]
|
||||
pub mod cma_es;
|
||||
pub mod grid;
|
||||
pub mod motpe;
|
||||
pub mod nsga2;
|
||||
pub mod random;
|
||||
#[cfg(feature = "sobol")]
|
||||
pub mod sobol;
|
||||
mod random;
|
||||
pub mod tpe;
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub use random::RandomSampler;
|
||||
pub use tpe::{TpeSampler, TpeSamplerBuilder};
|
||||
|
||||
use crate::distribution::Distribution;
|
||||
use crate::param::ParamValue;
|
||||
use crate::parameter::{ParamId, Parameter};
|
||||
use crate::trial::AttrValue;
|
||||
use crate::types::TrialState;
|
||||
|
||||
/// A completed trial with its parameters, distributions, and objective value.
|
||||
///
|
||||
@@ -25,167 +17,30 @@ use crate::types::TrialState;
|
||||
/// parameter values, their distributions, and the objective value returned
|
||||
/// by the objective function.
|
||||
#[derive(Clone, Debug)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
pub struct CompletedTrial<V = f64> {
|
||||
/// The unique identifier for this trial.
|
||||
pub id: u64,
|
||||
/// The sampled parameter values, keyed by parameter id.
|
||||
pub params: HashMap<ParamId, ParamValue>,
|
||||
/// The parameter distributions used, keyed by parameter id.
|
||||
pub distributions: HashMap<ParamId, Distribution>,
|
||||
/// Human-readable labels for parameters, keyed by parameter id.
|
||||
pub param_labels: HashMap<ParamId, String>,
|
||||
/// The sampled parameter values, keyed by parameter name.
|
||||
pub params: HashMap<String, ParamValue>,
|
||||
/// The parameter distributions used, keyed by parameter name.
|
||||
pub distributions: HashMap<String, Distribution>,
|
||||
/// The objective value returned by the objective function.
|
||||
pub value: V,
|
||||
/// Intermediate objective values reported during the trial.
|
||||
pub intermediate_values: Vec<(u64, f64)>,
|
||||
/// The state of the trial (Complete, Pruned, or Failed).
|
||||
pub state: TrialState,
|
||||
/// User-defined attributes stored during the trial.
|
||||
pub user_attrs: HashMap<String, AttrValue>,
|
||||
/// Constraint values for this trial (<=0.0 means feasible).
|
||||
#[cfg_attr(feature = "serde", serde(default))]
|
||||
pub constraints: Vec<f64>,
|
||||
}
|
||||
|
||||
impl<V> CompletedTrial<V> {
|
||||
/// Creates a new completed trial.
|
||||
pub fn new(
|
||||
id: u64,
|
||||
params: HashMap<ParamId, ParamValue>,
|
||||
distributions: HashMap<ParamId, Distribution>,
|
||||
param_labels: HashMap<ParamId, String>,
|
||||
params: HashMap<String, ParamValue>,
|
||||
distributions: HashMap<String, Distribution>,
|
||||
value: V,
|
||||
) -> Self {
|
||||
Self {
|
||||
id,
|
||||
params,
|
||||
distributions,
|
||||
param_labels,
|
||||
value,
|
||||
intermediate_values: Vec::new(),
|
||||
state: TrialState::Complete,
|
||||
user_attrs: HashMap::new(),
|
||||
constraints: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a new completed trial with intermediate values and user attributes.
|
||||
pub fn with_intermediate_values(
|
||||
id: u64,
|
||||
params: HashMap<ParamId, ParamValue>,
|
||||
distributions: HashMap<ParamId, Distribution>,
|
||||
param_labels: HashMap<ParamId, String>,
|
||||
value: V,
|
||||
intermediate_values: Vec<(u64, f64)>,
|
||||
user_attrs: HashMap<String, AttrValue>,
|
||||
) -> Self {
|
||||
Self {
|
||||
id,
|
||||
params,
|
||||
distributions,
|
||||
param_labels,
|
||||
value,
|
||||
intermediate_values,
|
||||
state: TrialState::Complete,
|
||||
user_attrs,
|
||||
constraints: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the typed value for the given parameter.
|
||||
///
|
||||
/// Looks up the parameter by its unique id and casts the stored
|
||||
/// [`ParamValue`] to the parameter's typed value.
|
||||
///
|
||||
/// Returns `None` if the parameter was not used in this trial.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// Panics if the stored value is incompatible with the parameter type
|
||||
/// (e.g., a `Float` value stored for an `IntParam`). This indicates
|
||||
/// a bug in the program, not a runtime error.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::parameter::{FloatParam, Parameter};
|
||||
/// use optimizer::{Direction, Study};
|
||||
///
|
||||
/// let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
/// let x = FloatParam::new(-10.0, 10.0);
|
||||
///
|
||||
/// study
|
||||
/// .optimize(5, |trial| {
|
||||
/// let val = x.suggest(trial)?;
|
||||
/// Ok::<_, optimizer::Error>(val * val)
|
||||
/// })
|
||||
/// .unwrap();
|
||||
///
|
||||
/// let best = study.best_trial().unwrap();
|
||||
/// let x_val: f64 = best.get(&x).unwrap();
|
||||
/// assert!((-10.0..=10.0).contains(&x_val));
|
||||
/// ```
|
||||
pub fn get<P: Parameter>(&self, param: &P) -> Option<P::Value> {
|
||||
self.params.get(¶m.id()).map(|v| {
|
||||
param
|
||||
.cast_param_value(v)
|
||||
.expect("parameter type mismatch: stored value incompatible with parameter")
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns `true` if all constraints are satisfied (values <= 0.0).
|
||||
///
|
||||
/// A trial with no constraints is considered feasible.
|
||||
#[must_use]
|
||||
pub fn is_feasible(&self) -> bool {
|
||||
self.constraints.iter().all(|&c| c <= 0.0)
|
||||
}
|
||||
|
||||
/// Gets a user attribute by key.
|
||||
#[must_use]
|
||||
pub fn user_attr(&self, key: &str) -> Option<&AttrValue> {
|
||||
self.user_attrs.get(key)
|
||||
}
|
||||
|
||||
/// Returns all user attributes.
|
||||
#[must_use]
|
||||
pub fn user_attrs(&self) -> &HashMap<String, AttrValue> {
|
||||
&self.user_attrs
|
||||
}
|
||||
}
|
||||
|
||||
/// A pending (running) trial with its parameters and distributions, but no objective value yet.
|
||||
///
|
||||
/// This struct represents a trial that has been started and has sampled parameters,
|
||||
/// but is still running and hasn't returned an objective value. It is used with the
|
||||
/// constant liar strategy for parallel optimization.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PendingTrial {
|
||||
/// The unique identifier for this trial.
|
||||
pub id: u64,
|
||||
/// The sampled parameter values, keyed by parameter id.
|
||||
pub params: HashMap<ParamId, ParamValue>,
|
||||
/// The parameter distributions used, keyed by parameter id.
|
||||
pub distributions: HashMap<ParamId, Distribution>,
|
||||
/// Human-readable labels for parameters, keyed by parameter id.
|
||||
pub param_labels: HashMap<ParamId, String>,
|
||||
}
|
||||
|
||||
impl PendingTrial {
|
||||
/// Creates a new pending trial.
|
||||
#[must_use]
|
||||
pub fn new(
|
||||
id: u64,
|
||||
params: HashMap<ParamId, ParamValue>,
|
||||
distributions: HashMap<ParamId, Distribution>,
|
||||
param_labels: HashMap<ParamId, String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
id,
|
||||
params,
|
||||
distributions,
|
||||
param_labels,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -195,6 +50,28 @@ impl PendingTrial {
|
||||
/// Samplers are responsible for generating parameter values based on
|
||||
/// the distribution and historical trial data. The trait requires
|
||||
/// `Send + Sync` to support concurrent and async optimization.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// Implementing a custom sampler:
|
||||
///
|
||||
/// ```ignore
|
||||
/// use optimizer::{Sampler, ParamValue, Distribution, CompletedTrial};
|
||||
///
|
||||
/// struct MySampler;
|
||||
///
|
||||
/// impl Sampler for MySampler {
|
||||
/// fn sample(
|
||||
/// &self,
|
||||
/// distribution: &Distribution,
|
||||
/// trial_id: u64,
|
||||
/// history: &[CompletedTrial],
|
||||
/// ) -> ParamValue {
|
||||
/// // Custom sampling logic here
|
||||
/// todo!()
|
||||
/// }
|
||||
/// }
|
||||
/// ```
|
||||
pub trait Sampler: Send + Sync {
|
||||
/// Samples a parameter value from the given distribution.
|
||||
///
|
||||
|
||||
@@ -1,932 +0,0 @@
|
||||
//! Multi-Objective Tree-Parzen Estimator (MOTPE) sampler.
|
||||
//!
|
||||
//! Extends TPE to handle multi-objective optimization by using Pareto
|
||||
//! non-dominated sorting to define "good" vs "bad" trial regions for
|
||||
//! the KDE models, replacing the single-objective gamma-based split.
|
||||
//!
|
||||
//! # Algorithm
|
||||
//!
|
||||
//! In single-objective TPE, trials are sorted by value and split at a
|
||||
//! gamma percentile into good/bad groups. MOTPE replaces this with:
|
||||
//!
|
||||
//! 1. Compute non-dominated sorting on all completed trials
|
||||
//! 2. Use the Pareto front (rank 0) as "good" trials
|
||||
//! 3. Use dominated trials as "bad" trials
|
||||
//! 4. Build KDE l(x) from good, g(x) from bad
|
||||
//! 5. Sample candidates and score by l(x)/g(x)
|
||||
//!
|
||||
//! # Examples
|
||||
//!
|
||||
//! ```
|
||||
//! use optimizer::Direction;
|
||||
//! use optimizer::multi_objective::MultiObjectiveStudy;
|
||||
//! use optimizer::parameter::{FloatParam, Parameter};
|
||||
//! use optimizer::sampler::motpe::MotpeSampler;
|
||||
//!
|
||||
//! let sampler = MotpeSampler::builder().seed(42).build();
|
||||
//! let study =
|
||||
//! MultiObjectiveStudy::with_sampler(vec![Direction::Minimize, Direction::Minimize], sampler);
|
||||
//!
|
||||
//! let x = FloatParam::new(0.0, 1.0);
|
||||
//! study
|
||||
//! .optimize(30, |trial| {
|
||||
//! let xv = x.suggest(trial)?;
|
||||
//! Ok::<_, optimizer::Error>(vec![xv, 1.0 - xv])
|
||||
//! })
|
||||
//! .unwrap();
|
||||
//!
|
||||
//! let front = study.pareto_front();
|
||||
//! assert!(!front.is_empty());
|
||||
//! ```
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use rand::rngs::StdRng;
|
||||
use rand::{RngExt, SeedableRng};
|
||||
|
||||
use crate::distribution::Distribution;
|
||||
use crate::kde::KernelDensityEstimator;
|
||||
use crate::multi_objective::{MultiObjectiveSampler, MultiObjectiveTrial};
|
||||
use crate::param::ParamValue;
|
||||
use crate::pareto;
|
||||
use crate::types::{Direction, TrialState};
|
||||
|
||||
/// Multi-Objective TPE (MOTPE) sampler for multi-objective Bayesian optimization.
|
||||
///
|
||||
/// Uses Pareto non-dominated sorting to split completed trials into
|
||||
/// "good" (non-dominated, rank 0) and "bad" (dominated) groups, then
|
||||
/// fits kernel density estimators to each group and samples new points
|
||||
/// that maximize l(x)/g(x).
|
||||
///
|
||||
/// During the startup phase (fewer than `n_startup_trials` completed),
|
||||
/// MOTPE falls back to random sampling.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::Direction;
|
||||
/// use optimizer::multi_objective::MultiObjectiveStudy;
|
||||
/// use optimizer::parameter::{FloatParam, Parameter};
|
||||
/// use optimizer::sampler::motpe::MotpeSampler;
|
||||
///
|
||||
/// let sampler = MotpeSampler::builder()
|
||||
/// .n_startup_trials(10)
|
||||
/// .n_ei_candidates(24)
|
||||
/// .seed(42)
|
||||
/// .build();
|
||||
///
|
||||
/// let study =
|
||||
/// MultiObjectiveStudy::with_sampler(vec![Direction::Minimize, Direction::Minimize], sampler);
|
||||
/// ```
|
||||
pub struct MotpeSampler {
|
||||
/// Number of trials before MOTPE kicks in (uses random sampling before this).
|
||||
n_startup_trials: usize,
|
||||
/// Number of candidate samples to evaluate when selecting the next point.
|
||||
n_ei_candidates: usize,
|
||||
/// Optional fixed bandwidth for KDE. If None, uses Scott's rule.
|
||||
kde_bandwidth: Option<f64>,
|
||||
/// Thread-safe RNG for sampling.
|
||||
rng: Mutex<StdRng>,
|
||||
}
|
||||
|
||||
impl MotpeSampler {
|
||||
/// Creates a new MOTPE sampler with default settings.
|
||||
///
|
||||
/// Defaults:
|
||||
/// - `n_startup_trials`: 11
|
||||
/// - `n_ei_candidates`: 24
|
||||
/// - `kde_bandwidth`: None (Scott's rule)
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
n_startup_trials: 11,
|
||||
n_ei_candidates: 24,
|
||||
kde_bandwidth: None,
|
||||
rng: Mutex::new(rand::make_rng()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a new MOTPE sampler with a fixed seed.
|
||||
#[must_use]
|
||||
pub fn with_seed(seed: u64) -> Self {
|
||||
Self {
|
||||
n_startup_trials: 11,
|
||||
n_ei_candidates: 24,
|
||||
kde_bandwidth: None,
|
||||
rng: Mutex::new(StdRng::seed_from_u64(seed)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a builder for configuring a MOTPE sampler.
|
||||
#[must_use]
|
||||
pub fn builder() -> MotpeSamplerBuilder {
|
||||
MotpeSamplerBuilder::new()
|
||||
}
|
||||
|
||||
/// Splits trials into good (non-dominated) and bad (dominated) groups
|
||||
/// using Pareto non-dominated sorting.
|
||||
fn split_trials<'a>(
|
||||
history: &'a [MultiObjectiveTrial],
|
||||
directions: &[Direction],
|
||||
) -> (Vec<&'a MultiObjectiveTrial>, Vec<&'a MultiObjectiveTrial>) {
|
||||
let complete: Vec<(usize, &MultiObjectiveTrial)> = history
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, t)| t.state == TrialState::Complete)
|
||||
.collect();
|
||||
|
||||
if complete.is_empty() {
|
||||
return (vec![], vec![]);
|
||||
}
|
||||
|
||||
let values: Vec<Vec<f64>> = complete.iter().map(|(_, t)| t.values.clone()).collect();
|
||||
let constraints: Vec<Vec<f64>> = complete
|
||||
.iter()
|
||||
.map(|(_, t)| t.constraints.clone())
|
||||
.collect();
|
||||
let has_constraints = constraints.iter().any(|c| !c.is_empty());
|
||||
|
||||
let fronts = if has_constraints {
|
||||
pareto::fast_non_dominated_sort_constrained(&values, directions, &constraints)
|
||||
} else {
|
||||
pareto::fast_non_dominated_sort(&values, directions)
|
||||
};
|
||||
|
||||
if fronts.is_empty() {
|
||||
return (vec![], vec![]);
|
||||
}
|
||||
|
||||
// Front 0 = good (non-dominated), everything else = bad
|
||||
let good: Vec<&MultiObjectiveTrial> = fronts[0].iter().map(|&i| complete[i].1).collect();
|
||||
|
||||
let bad: Vec<&MultiObjectiveTrial> = fronts[1..]
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(|&i| complete[i].1)
|
||||
.collect();
|
||||
|
||||
(good, bad)
|
||||
}
|
||||
|
||||
/// Samples uniformly from a distribution (used during startup phase).
|
||||
#[allow(
|
||||
clippy::cast_possible_truncation,
|
||||
clippy::cast_precision_loss,
|
||||
clippy::unused_self
|
||||
)]
|
||||
fn sample_uniform(distribution: &Distribution, rng: &mut StdRng) -> ParamValue {
|
||||
match distribution {
|
||||
Distribution::Float(d) => {
|
||||
let value = if d.log_scale {
|
||||
let log_low = d.low.ln();
|
||||
let log_high = d.high.ln();
|
||||
rng.random_range(log_low..=log_high).exp()
|
||||
} else if let Some(step) = d.step {
|
||||
let n_steps = ((d.high - d.low) / step).floor() as i64;
|
||||
let k = rng.random_range(0..=n_steps);
|
||||
d.low + (k as f64) * step
|
||||
} else {
|
||||
rng.random_range(d.low..=d.high)
|
||||
};
|
||||
ParamValue::Float(value)
|
||||
}
|
||||
Distribution::Int(d) => {
|
||||
let value = if d.log_scale {
|
||||
let log_low = (d.low as f64).ln();
|
||||
let log_high = (d.high as f64).ln();
|
||||
let raw = rng.random_range(log_low..=log_high).exp().round() as i64;
|
||||
raw.clamp(d.low, d.high)
|
||||
} else if let Some(step) = d.step {
|
||||
let n_steps = (d.high - d.low) / step;
|
||||
let k = rng.random_range(0..=n_steps);
|
||||
d.low + k * step
|
||||
} else {
|
||||
rng.random_range(d.low..=d.high)
|
||||
};
|
||||
ParamValue::Int(value)
|
||||
}
|
||||
Distribution::Categorical(d) => {
|
||||
ParamValue::Categorical(rng.random_range(0..d.n_choices))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Samples using TPE for float distributions.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn sample_tpe_float(
|
||||
&self,
|
||||
low: f64,
|
||||
high: f64,
|
||||
log_scale: bool,
|
||||
step: Option<f64>,
|
||||
good_values: Vec<f64>,
|
||||
bad_values: Vec<f64>,
|
||||
rng: &mut StdRng,
|
||||
) -> f64 {
|
||||
// Transform to internal space (log space if needed)
|
||||
let (internal_low, internal_high, good_internal, bad_internal) = if log_scale {
|
||||
let i_low = low.ln();
|
||||
let i_high = high.ln();
|
||||
let g: Vec<f64> = good_values.iter().map(|&v| v.ln()).collect();
|
||||
let b: Vec<f64> = bad_values.iter().map(|&v| v.ln()).collect();
|
||||
(i_low, i_high, g, b)
|
||||
} else {
|
||||
(low, high, good_values, bad_values)
|
||||
};
|
||||
|
||||
// Fit KDEs to good and bad groups
|
||||
let l_kde = match self.kde_bandwidth {
|
||||
Some(bw) => KernelDensityEstimator::with_bandwidth(good_internal, bw),
|
||||
None => KernelDensityEstimator::new(good_internal),
|
||||
};
|
||||
let g_kde = match self.kde_bandwidth {
|
||||
Some(bw) => KernelDensityEstimator::with_bandwidth(bad_internal, bw),
|
||||
None => KernelDensityEstimator::new(bad_internal),
|
||||
};
|
||||
|
||||
// If KDE construction fails, fall back to uniform sampling
|
||||
let (Ok(l_kde), Ok(g_kde)) = (l_kde, g_kde) else {
|
||||
return rng.random_range(low..=high);
|
||||
};
|
||||
|
||||
// Generate candidates from l(x) and select the one with best l(x)/g(x)
|
||||
let mut best_candidate = internal_low;
|
||||
let mut best_ratio = f64::NEG_INFINITY;
|
||||
|
||||
for _ in 0..self.n_ei_candidates {
|
||||
let candidate = l_kde.sample(rng).clamp(internal_low, internal_high);
|
||||
|
||||
let l_density = l_kde.pdf(candidate);
|
||||
let g_density = g_kde.pdf(candidate);
|
||||
|
||||
let ratio = if g_density < f64::EPSILON {
|
||||
if l_density > f64::EPSILON {
|
||||
f64::INFINITY
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
} else {
|
||||
l_density / g_density
|
||||
};
|
||||
|
||||
if ratio > best_ratio {
|
||||
best_ratio = ratio;
|
||||
best_candidate = candidate;
|
||||
}
|
||||
}
|
||||
|
||||
// Transform back from internal space
|
||||
let mut value = if log_scale {
|
||||
best_candidate.exp()
|
||||
} else {
|
||||
best_candidate
|
||||
};
|
||||
|
||||
// Apply step constraint if present
|
||||
if let Some(step) = step {
|
||||
let k = ((value - low) / step).round();
|
||||
value = low + k * step;
|
||||
}
|
||||
|
||||
value.clamp(low, high)
|
||||
}
|
||||
|
||||
/// Samples using TPE for integer distributions.
|
||||
#[allow(
|
||||
clippy::too_many_arguments,
|
||||
clippy::cast_precision_loss,
|
||||
clippy::cast_possible_truncation
|
||||
)]
|
||||
fn sample_tpe_int(
|
||||
&self,
|
||||
low: i64,
|
||||
high: i64,
|
||||
log_scale: bool,
|
||||
step: Option<i64>,
|
||||
good_values: &[i64],
|
||||
bad_values: &[i64],
|
||||
rng: &mut StdRng,
|
||||
) -> i64 {
|
||||
let good_floats: Vec<f64> = good_values.iter().map(|&v| v as f64).collect();
|
||||
let bad_floats: Vec<f64> = bad_values.iter().map(|&v| v as f64).collect();
|
||||
|
||||
let float_value = self.sample_tpe_float(
|
||||
low as f64,
|
||||
high as f64,
|
||||
log_scale,
|
||||
step.map(|s| s as f64),
|
||||
good_floats,
|
||||
bad_floats,
|
||||
rng,
|
||||
);
|
||||
|
||||
let int_value = float_value.round() as i64;
|
||||
let int_value = if let Some(step) = step {
|
||||
let k = ((int_value - low) as f64 / step as f64).round() as i64;
|
||||
low + k * step
|
||||
} else {
|
||||
int_value
|
||||
};
|
||||
|
||||
int_value.clamp(low, high)
|
||||
}
|
||||
|
||||
/// Samples using TPE for categorical distributions.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
fn sample_tpe_categorical(
|
||||
n_choices: usize,
|
||||
good_indices: &[usize],
|
||||
bad_indices: &[usize],
|
||||
rng: &mut StdRng,
|
||||
) -> usize {
|
||||
let mut good_counts = vec![0usize; n_choices];
|
||||
let mut bad_counts = vec![0usize; n_choices];
|
||||
|
||||
for &idx in good_indices {
|
||||
if idx < n_choices {
|
||||
good_counts[idx] += 1;
|
||||
}
|
||||
}
|
||||
for &idx in bad_indices {
|
||||
if idx < n_choices {
|
||||
bad_counts[idx] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Laplace smoothing
|
||||
let good_total = good_indices.len() as f64 + n_choices as f64;
|
||||
let bad_total = bad_indices.len() as f64 + n_choices as f64;
|
||||
|
||||
let mut weights = vec![0.0f64; n_choices];
|
||||
for i in 0..n_choices {
|
||||
let l_prob = (good_counts[i] as f64 + 1.0) / good_total;
|
||||
let g_prob = (bad_counts[i] as f64 + 1.0) / bad_total;
|
||||
weights[i] = l_prob / g_prob;
|
||||
}
|
||||
|
||||
// Sample proportionally to weights
|
||||
let total_weight: f64 = weights.iter().sum();
|
||||
let threshold = rng.random::<f64>() * total_weight;
|
||||
|
||||
let mut cumulative = 0.0;
|
||||
for (i, &w) in weights.iter().enumerate() {
|
||||
cumulative += w;
|
||||
if cumulative >= threshold {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
|
||||
n_choices - 1
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for MotpeSampler {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl MultiObjectiveSampler for MotpeSampler {
|
||||
#[allow(clippy::too_many_lines)]
|
||||
fn sample(
|
||||
&self,
|
||||
distribution: &Distribution,
|
||||
_trial_id: u64,
|
||||
history: &[MultiObjectiveTrial],
|
||||
directions: &[Direction],
|
||||
) -> ParamValue {
|
||||
let mut rng = self.rng.lock();
|
||||
|
||||
// Fall back to random sampling during startup phase
|
||||
let n_complete = history
|
||||
.iter()
|
||||
.filter(|t| t.state == TrialState::Complete)
|
||||
.count();
|
||||
if n_complete < self.n_startup_trials {
|
||||
return Self::sample_uniform(distribution, &mut rng);
|
||||
}
|
||||
|
||||
// Split trials into good (Pareto front) and bad (dominated)
|
||||
let (good_trials, bad_trials) = Self::split_trials(history, directions);
|
||||
|
||||
if good_trials.is_empty() || bad_trials.is_empty() {
|
||||
return Self::sample_uniform(distribution, &mut rng);
|
||||
}
|
||||
|
||||
match distribution {
|
||||
Distribution::Float(d) => {
|
||||
let good_values: Vec<f64> = good_trials
|
||||
.iter()
|
||||
.flat_map(|t| t.params.values())
|
||||
.filter_map(|v| match v {
|
||||
ParamValue::Float(f) => Some(*f),
|
||||
_ => None,
|
||||
})
|
||||
.filter(|&v| v >= d.low && v <= d.high)
|
||||
.collect();
|
||||
|
||||
let bad_values: Vec<f64> = bad_trials
|
||||
.iter()
|
||||
.flat_map(|t| t.params.values())
|
||||
.filter_map(|v| match v {
|
||||
ParamValue::Float(f) => Some(*f),
|
||||
_ => None,
|
||||
})
|
||||
.filter(|&v| v >= d.low && v <= d.high)
|
||||
.collect();
|
||||
|
||||
if good_values.is_empty() || bad_values.is_empty() {
|
||||
return Self::sample_uniform(distribution, &mut rng);
|
||||
}
|
||||
|
||||
let value = self.sample_tpe_float(
|
||||
d.low,
|
||||
d.high,
|
||||
d.log_scale,
|
||||
d.step,
|
||||
good_values,
|
||||
bad_values,
|
||||
&mut rng,
|
||||
);
|
||||
ParamValue::Float(value)
|
||||
}
|
||||
Distribution::Int(d) => {
|
||||
let good_values: Vec<i64> = good_trials
|
||||
.iter()
|
||||
.flat_map(|t| t.params.values())
|
||||
.filter_map(|v| match v {
|
||||
ParamValue::Int(i) => Some(*i),
|
||||
_ => None,
|
||||
})
|
||||
.filter(|&v| v >= d.low && v <= d.high)
|
||||
.collect();
|
||||
|
||||
let bad_values: Vec<i64> = bad_trials
|
||||
.iter()
|
||||
.flat_map(|t| t.params.values())
|
||||
.filter_map(|v| match v {
|
||||
ParamValue::Int(i) => Some(*i),
|
||||
_ => None,
|
||||
})
|
||||
.filter(|&v| v >= d.low && v <= d.high)
|
||||
.collect();
|
||||
|
||||
if good_values.is_empty() || bad_values.is_empty() {
|
||||
return Self::sample_uniform(distribution, &mut rng);
|
||||
}
|
||||
|
||||
let value = self.sample_tpe_int(
|
||||
d.low,
|
||||
d.high,
|
||||
d.log_scale,
|
||||
d.step,
|
||||
&good_values,
|
||||
&bad_values,
|
||||
&mut rng,
|
||||
);
|
||||
ParamValue::Int(value)
|
||||
}
|
||||
Distribution::Categorical(d) => {
|
||||
let good_indices: Vec<usize> = good_trials
|
||||
.iter()
|
||||
.flat_map(|t| t.params.values())
|
||||
.filter_map(|v| match v {
|
||||
ParamValue::Categorical(i) => Some(*i),
|
||||
_ => None,
|
||||
})
|
||||
.filter(|&i| i < d.n_choices)
|
||||
.collect();
|
||||
|
||||
let bad_indices: Vec<usize> = bad_trials
|
||||
.iter()
|
||||
.flat_map(|t| t.params.values())
|
||||
.filter_map(|v| match v {
|
||||
ParamValue::Categorical(i) => Some(*i),
|
||||
_ => None,
|
||||
})
|
||||
.filter(|&i| i < d.n_choices)
|
||||
.collect();
|
||||
|
||||
if good_indices.is_empty() || bad_indices.is_empty() {
|
||||
return Self::sample_uniform(distribution, &mut rng);
|
||||
}
|
||||
|
||||
let index = Self::sample_tpe_categorical(
|
||||
d.n_choices,
|
||||
&good_indices,
|
||||
&bad_indices,
|
||||
&mut rng,
|
||||
);
|
||||
ParamValue::Categorical(index)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Builder for configuring a [`MotpeSampler`].
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::sampler::motpe::MotpeSamplerBuilder;
|
||||
///
|
||||
/// let sampler = MotpeSamplerBuilder::new()
|
||||
/// .n_startup_trials(15)
|
||||
/// .n_ei_candidates(32)
|
||||
/// .seed(42)
|
||||
/// .build();
|
||||
/// ```
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MotpeSamplerBuilder {
|
||||
n_startup_trials: usize,
|
||||
n_ei_candidates: usize,
|
||||
kde_bandwidth: Option<f64>,
|
||||
seed: Option<u64>,
|
||||
}
|
||||
|
||||
impl MotpeSamplerBuilder {
|
||||
/// Creates a new builder with default settings.
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
n_startup_trials: 11,
|
||||
n_ei_candidates: 24,
|
||||
kde_bandwidth: None,
|
||||
seed: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets the number of startup trials before MOTPE sampling begins.
|
||||
#[must_use]
|
||||
pub fn n_startup_trials(mut self, n: usize) -> Self {
|
||||
self.n_startup_trials = n;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the number of EI candidates to evaluate per sample.
|
||||
#[must_use]
|
||||
pub fn n_ei_candidates(mut self, n: usize) -> Self {
|
||||
self.n_ei_candidates = n;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets a fixed bandwidth for the kernel density estimator.
|
||||
///
|
||||
/// By default, Scott's rule is used for automatic bandwidth selection.
|
||||
#[must_use]
|
||||
pub fn kde_bandwidth(mut self, bandwidth: f64) -> Self {
|
||||
self.kde_bandwidth = Some(bandwidth);
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets a seed for reproducible sampling.
|
||||
#[must_use]
|
||||
pub fn seed(mut self, seed: u64) -> Self {
|
||||
self.seed = Some(seed);
|
||||
self
|
||||
}
|
||||
|
||||
/// Builds the configured [`MotpeSampler`].
|
||||
#[must_use]
|
||||
pub fn build(self) -> MotpeSampler {
|
||||
let rng = match self.seed {
|
||||
Some(s) => StdRng::seed_from_u64(s),
|
||||
None => rand::make_rng(),
|
||||
};
|
||||
|
||||
MotpeSampler {
|
||||
n_startup_trials: self.n_startup_trials,
|
||||
n_ei_candidates: self.n_ei_candidates,
|
||||
kde_bandwidth: self.kde_bandwidth,
|
||||
rng: Mutex::new(rng),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for MotpeSamplerBuilder {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(
|
||||
clippy::similar_names,
|
||||
clippy::cast_sign_loss,
|
||||
clippy::cast_precision_loss
|
||||
)]
|
||||
mod tests {
|
||||
use std::collections::HashMap;
|
||||
|
||||
use super::*;
|
||||
use crate::distribution::{CategoricalDistribution, FloatDistribution, IntDistribution};
|
||||
use crate::parameter::ParamId;
|
||||
|
||||
fn create_mo_trial(
|
||||
id: u64,
|
||||
values: Vec<f64>,
|
||||
params: Vec<(ParamId, ParamValue, Distribution)>,
|
||||
) -> MultiObjectiveTrial {
|
||||
let mut param_map = HashMap::new();
|
||||
let mut dist_map = HashMap::new();
|
||||
let label_map = HashMap::new();
|
||||
for (param_id, pv, dist) in params {
|
||||
param_map.insert(param_id, pv);
|
||||
dist_map.insert(param_id, dist);
|
||||
}
|
||||
MultiObjectiveTrial {
|
||||
id,
|
||||
params: param_map,
|
||||
distributions: dist_map,
|
||||
param_labels: label_map,
|
||||
values,
|
||||
state: TrialState::Complete,
|
||||
user_attrs: HashMap::new(),
|
||||
constraints: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_motpe_startup_random_sampling() {
|
||||
let sampler = MotpeSampler::with_seed(42);
|
||||
let dist = Distribution::Float(FloatDistribution {
|
||||
low: 0.0,
|
||||
high: 1.0,
|
||||
log_scale: false,
|
||||
step: None,
|
||||
});
|
||||
let directions = [Direction::Minimize, Direction::Minimize];
|
||||
|
||||
// With no history, should use random sampling
|
||||
let history: Vec<MultiObjectiveTrial> = vec![];
|
||||
for _ in 0..50 {
|
||||
let value = sampler.sample(&dist, 0, &history, &directions);
|
||||
if let ParamValue::Float(v) = value {
|
||||
assert!((0.0..=1.0).contains(&v));
|
||||
} else {
|
||||
panic!("Expected Float value");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_motpe_split_pareto() {
|
||||
let dist = Distribution::Float(FloatDistribution {
|
||||
low: 0.0,
|
||||
high: 1.0,
|
||||
log_scale: false,
|
||||
step: None,
|
||||
});
|
||||
let directions = [Direction::Minimize, Direction::Minimize];
|
||||
let x_id = ParamId::new();
|
||||
|
||||
// Create trials: Pareto front = {(0.1, 0.9), (0.5, 0.5), (0.9, 0.1)}
|
||||
// Dominated = {(0.6, 0.8), (0.8, 0.7)}
|
||||
let history = vec![
|
||||
create_mo_trial(
|
||||
0,
|
||||
vec![0.1, 0.9],
|
||||
vec![(x_id, ParamValue::Float(0.1), dist.clone())],
|
||||
),
|
||||
create_mo_trial(
|
||||
1,
|
||||
vec![0.5, 0.5],
|
||||
vec![(x_id, ParamValue::Float(0.5), dist.clone())],
|
||||
),
|
||||
create_mo_trial(
|
||||
2,
|
||||
vec![0.9, 0.1],
|
||||
vec![(x_id, ParamValue::Float(0.9), dist.clone())],
|
||||
),
|
||||
create_mo_trial(
|
||||
3,
|
||||
vec![0.6, 0.8],
|
||||
vec![(x_id, ParamValue::Float(0.6), dist.clone())],
|
||||
),
|
||||
create_mo_trial(
|
||||
4,
|
||||
vec![0.8, 0.7],
|
||||
vec![(x_id, ParamValue::Float(0.8), dist.clone())],
|
||||
),
|
||||
];
|
||||
|
||||
let (good, bad) = MotpeSampler::split_trials(&history, &directions);
|
||||
assert_eq!(good.len(), 3, "Pareto front should have 3 members");
|
||||
assert_eq!(bad.len(), 2, "2 dominated trials");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_motpe_samples_float() {
|
||||
let sampler = MotpeSampler::builder()
|
||||
.n_startup_trials(5)
|
||||
.n_ei_candidates(24)
|
||||
.seed(42)
|
||||
.build();
|
||||
|
||||
let dist = Distribution::Float(FloatDistribution {
|
||||
low: 0.0,
|
||||
high: 1.0,
|
||||
log_scale: false,
|
||||
step: None,
|
||||
});
|
||||
let directions = [Direction::Minimize, Direction::Minimize];
|
||||
let x_id = ParamId::new();
|
||||
|
||||
// Build history where values near 0.3 are on the Pareto front
|
||||
let mut history = Vec::new();
|
||||
for i in 0..20 {
|
||||
let x = f64::from(i) / 20.0;
|
||||
// Pareto front: f1 = (x - 0.3)^2, f2 = (x - 0.3)^2 + 0.1
|
||||
// Best solutions cluster around x = 0.3
|
||||
let f1 = (x - 0.3).powi(2);
|
||||
let f2 = (x - 0.7).powi(2);
|
||||
history.push(create_mo_trial(
|
||||
i as u64,
|
||||
vec![f1, f2],
|
||||
vec![(x_id, ParamValue::Float(x), dist.clone())],
|
||||
));
|
||||
}
|
||||
|
||||
// MOTPE should produce values within [0, 1]
|
||||
for i in 0..50 {
|
||||
let value = sampler.sample(&dist, 100 + i, &history, &directions);
|
||||
if let ParamValue::Float(v) = value {
|
||||
assert!((0.0..=1.0).contains(&v), "Value {v} out of range");
|
||||
} else {
|
||||
panic!("Expected Float value");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_motpe_int_sampling() {
|
||||
let sampler = MotpeSampler::builder().n_startup_trials(5).seed(42).build();
|
||||
|
||||
let dist = Distribution::Int(IntDistribution {
|
||||
low: 0,
|
||||
high: 100,
|
||||
log_scale: false,
|
||||
step: None,
|
||||
});
|
||||
let directions = [Direction::Minimize, Direction::Minimize];
|
||||
let x_id = ParamId::new();
|
||||
|
||||
let mut history = Vec::new();
|
||||
for i in 0..20 {
|
||||
let x = i * 5;
|
||||
let f1 = ((x as f64) - 30.0).powi(2);
|
||||
let f2 = ((x as f64) - 70.0).powi(2);
|
||||
history.push(create_mo_trial(
|
||||
i as u64,
|
||||
vec![f1, f2],
|
||||
vec![(x_id, ParamValue::Int(x), dist.clone())],
|
||||
));
|
||||
}
|
||||
|
||||
for i in 0..50 {
|
||||
let value = sampler.sample(&dist, 100 + i, &history, &directions);
|
||||
if let ParamValue::Int(v) = value {
|
||||
assert!((0..=100).contains(&v), "Value {v} out of range");
|
||||
} else {
|
||||
panic!("Expected Int value");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_motpe_categorical_sampling() {
|
||||
let sampler = MotpeSampler::builder().n_startup_trials(5).seed(42).build();
|
||||
|
||||
let dist = Distribution::Categorical(CategoricalDistribution { n_choices: 3 });
|
||||
let directions = [Direction::Minimize, Direction::Minimize];
|
||||
let cat_id = ParamId::new();
|
||||
|
||||
// Category 1 is on the Pareto front, others are dominated
|
||||
let mut history = Vec::new();
|
||||
for i in 0..15 {
|
||||
let category = i % 3;
|
||||
let (f1, f2) = match category {
|
||||
0 => (0.8, 0.8), // dominated
|
||||
1 => (0.1, 0.9), // Pareto front
|
||||
2 => (0.9, 0.1), // Pareto front
|
||||
_ => unreachable!(),
|
||||
};
|
||||
history.push(create_mo_trial(
|
||||
i as u64,
|
||||
vec![f1, f2],
|
||||
vec![(
|
||||
cat_id,
|
||||
ParamValue::Categorical(category as usize),
|
||||
dist.clone(),
|
||||
)],
|
||||
));
|
||||
}
|
||||
|
||||
let mut counts = vec![0usize; 3];
|
||||
for i in 0..200 {
|
||||
let value = sampler.sample(&dist, 100 + i, &history, &directions);
|
||||
if let ParamValue::Categorical(idx) = value {
|
||||
assert!(idx < 3, "Category {idx} out of range");
|
||||
counts[idx] += 1;
|
||||
} else {
|
||||
panic!("Expected Categorical value");
|
||||
}
|
||||
}
|
||||
|
||||
// Categories 1 and 2 (on Pareto front) should dominate category 0
|
||||
assert!(
|
||||
counts[1] + counts[2] > counts[0],
|
||||
"Pareto-front categories should be sampled more: {counts:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_motpe_reproducibility() {
|
||||
let dist = Distribution::Float(FloatDistribution {
|
||||
low: 0.0,
|
||||
high: 1.0,
|
||||
log_scale: false,
|
||||
step: None,
|
||||
});
|
||||
let directions = [Direction::Minimize, Direction::Minimize];
|
||||
let x_id = ParamId::new();
|
||||
|
||||
let history: Vec<MultiObjectiveTrial> = (0..20)
|
||||
.map(|i| {
|
||||
let x = f64::from(i) / 20.0;
|
||||
create_mo_trial(
|
||||
i as u64,
|
||||
vec![x, 1.0 - x],
|
||||
vec![(x_id, ParamValue::Float(x), dist.clone())],
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
|
||||
let sampler1 = MotpeSampler::builder()
|
||||
.seed(12345)
|
||||
.n_startup_trials(5)
|
||||
.build();
|
||||
let sampler2 = MotpeSampler::builder()
|
||||
.seed(12345)
|
||||
.n_startup_trials(5)
|
||||
.build();
|
||||
|
||||
for i in 0..10 {
|
||||
let v1 = sampler1.sample(&dist, i, &history, &directions);
|
||||
let v2 = sampler2.sample(&dist, i, &history, &directions);
|
||||
assert_eq!(v1, v2, "Samples should be identical with same seed");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_motpe_with_study() {
|
||||
use crate::multi_objective::MultiObjectiveStudy;
|
||||
use crate::parameter::{FloatParam, Parameter};
|
||||
|
||||
let sampler = MotpeSampler::builder().seed(42).build();
|
||||
let study = MultiObjectiveStudy::with_sampler(
|
||||
vec![Direction::Minimize, Direction::Minimize],
|
||||
sampler,
|
||||
);
|
||||
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
study
|
||||
.optimize(30, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
Ok::<_, crate::Error>(vec![xv, 1.0 - xv])
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let front = study.pareto_front();
|
||||
assert!(!front.is_empty(), "Should have Pareto-optimal solutions");
|
||||
|
||||
// All front solutions should have values summing to ~1.0
|
||||
for trial in &front {
|
||||
let sum: f64 = trial.values.iter().sum();
|
||||
assert!(
|
||||
(sum - 1.0).abs() < 0.01,
|
||||
"Pareto front values should sum to ~1.0, got {sum}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_motpe_builder_defaults() {
|
||||
let sampler = MotpeSamplerBuilder::new().build();
|
||||
assert_eq!(sampler.n_startup_trials, 11);
|
||||
assert_eq!(sampler.n_ei_candidates, 24);
|
||||
assert!(sampler.kde_bandwidth.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_motpe_builder_custom() {
|
||||
let sampler = MotpeSamplerBuilder::new()
|
||||
.n_startup_trials(20)
|
||||
.n_ei_candidates(48)
|
||||
.kde_bandwidth(0.5)
|
||||
.seed(99)
|
||||
.build();
|
||||
assert_eq!(sampler.n_startup_trials, 20);
|
||||
assert_eq!(sampler.n_ei_candidates, 48);
|
||||
assert_eq!(sampler.kde_bandwidth, Some(0.5));
|
||||
}
|
||||
}
|
||||
@@ -1,763 +0,0 @@
|
||||
//! NSGA-II (Non-dominated Sorting Genetic Algorithm II) sampler.
|
||||
//!
|
||||
//! Implements multi-objective optimization using non-dominated sorting,
|
||||
//! crowding distance, SBX crossover, and polynomial mutation.
|
||||
//!
|
||||
//! # Examples
|
||||
//!
|
||||
//! ```
|
||||
//! use optimizer::Direction;
|
||||
//! use optimizer::multi_objective::MultiObjectiveStudy;
|
||||
//! use optimizer::parameter::{FloatParam, Parameter};
|
||||
//! use optimizer::sampler::nsga2::Nsga2Sampler;
|
||||
//!
|
||||
//! let sampler = Nsga2Sampler::with_seed(42);
|
||||
//! let study =
|
||||
//! MultiObjectiveStudy::with_sampler(vec![Direction::Minimize, Direction::Minimize], sampler);
|
||||
//!
|
||||
//! let x = FloatParam::new(0.0, 1.0);
|
||||
//! study
|
||||
//! .optimize(50, |trial| {
|
||||
//! let xv = x.suggest(trial)?;
|
||||
//! Ok::<_, optimizer::Error>(vec![xv * xv, (xv - 1.0).powi(2)])
|
||||
//! })
|
||||
//! .unwrap();
|
||||
//! ```
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use rand::rngs::StdRng;
|
||||
use rand::{RngExt, SeedableRng};
|
||||
|
||||
use crate::distribution::Distribution;
|
||||
use crate::multi_objective::MultiObjectiveTrial;
|
||||
use crate::param::ParamValue;
|
||||
use crate::pareto;
|
||||
use crate::types::Direction;
|
||||
|
||||
/// NSGA-II sampler for multi-objective optimization.
|
||||
///
|
||||
/// Provides non-dominated sorting, crowding distance selection,
|
||||
/// SBX crossover, and polynomial mutation.
|
||||
pub struct Nsga2Sampler {
|
||||
state: Mutex<Nsga2State>,
|
||||
}
|
||||
|
||||
impl Nsga2Sampler {
|
||||
/// Creates a new NSGA-II sampler with a random seed.
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
state: Mutex::new(Nsga2State::new(Nsga2Config::default(), None)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a new NSGA-II sampler with a fixed seed.
|
||||
#[must_use]
|
||||
pub fn with_seed(seed: u64) -> Self {
|
||||
Self {
|
||||
state: Mutex::new(Nsga2State::new(Nsga2Config::default(), Some(seed))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a builder for configuring an `Nsga2Sampler`.
|
||||
#[must_use]
|
||||
pub fn builder() -> Nsga2SamplerBuilder {
|
||||
Nsga2SamplerBuilder::default()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Nsga2Sampler {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Builder for [`Nsga2Sampler`].
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct Nsga2SamplerBuilder {
|
||||
population_size: Option<usize>,
|
||||
crossover_prob: Option<f64>,
|
||||
crossover_eta: Option<f64>,
|
||||
mutation_eta: Option<f64>,
|
||||
seed: Option<u64>,
|
||||
}
|
||||
|
||||
impl Nsga2SamplerBuilder {
|
||||
/// Sets the population size. Default: `4 + floor(3 * ln(n_params))`, minimum 4.
|
||||
#[must_use]
|
||||
pub fn population_size(mut self, size: usize) -> Self {
|
||||
self.population_size = Some(size);
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the crossover probability. Default: 0.9.
|
||||
#[must_use]
|
||||
pub fn crossover_prob(mut self, prob: f64) -> Self {
|
||||
self.crossover_prob = Some(prob);
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the SBX distribution index. Default: 20.0.
|
||||
#[must_use]
|
||||
pub fn crossover_eta(mut self, eta: f64) -> Self {
|
||||
self.crossover_eta = Some(eta);
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the polynomial mutation distribution index. Default: 20.0.
|
||||
#[must_use]
|
||||
pub fn mutation_eta(mut self, eta: f64) -> Self {
|
||||
self.mutation_eta = Some(eta);
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the random seed for reproducibility.
|
||||
#[must_use]
|
||||
pub fn seed(mut self, seed: u64) -> Self {
|
||||
self.seed = Some(seed);
|
||||
self
|
||||
}
|
||||
|
||||
/// Builds the configured [`Nsga2Sampler`].
|
||||
#[must_use]
|
||||
pub fn build(self) -> Nsga2Sampler {
|
||||
let config = Nsga2Config {
|
||||
user_population_size: self.population_size,
|
||||
crossover_prob: self.crossover_prob.unwrap_or(0.9),
|
||||
crossover_eta: self.crossover_eta.unwrap_or(20.0),
|
||||
mutation_eta: self.mutation_eta.unwrap_or(20.0),
|
||||
};
|
||||
Nsga2Sampler {
|
||||
state: Mutex::new(Nsga2State::new(config, self.seed)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Internal types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct Nsga2Config {
|
||||
user_population_size: Option<usize>,
|
||||
crossover_prob: f64,
|
||||
crossover_eta: f64,
|
||||
mutation_eta: f64,
|
||||
}
|
||||
|
||||
impl Default for Nsga2Config {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
user_population_size: None,
|
||||
crossover_prob: 0.9,
|
||||
crossover_eta: 20.0,
|
||||
mutation_eta: 20.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Describes a parameter dimension.
|
||||
#[derive(Clone, Debug)]
|
||||
struct DimensionInfo {
|
||||
distribution: Distribution,
|
||||
}
|
||||
|
||||
/// A candidate solution: one value per dimension.
|
||||
#[derive(Clone, Debug)]
|
||||
struct Candidate {
|
||||
params: Vec<ParamValue>,
|
||||
}
|
||||
|
||||
/// Tracks per-trial sampling progress.
|
||||
#[derive(Clone, Debug)]
|
||||
struct TrialProgress {
|
||||
candidate_idx: usize,
|
||||
next_dim: usize,
|
||||
}
|
||||
|
||||
enum Phase {
|
||||
/// First trial reveals parameter dimensions.
|
||||
Discovery,
|
||||
/// NSGA-II optimisation.
|
||||
Active,
|
||||
}
|
||||
|
||||
struct Nsga2State {
|
||||
rng: StdRng,
|
||||
config: Nsga2Config,
|
||||
phase: Phase,
|
||||
dimensions: Vec<DimensionInfo>,
|
||||
population_size: usize,
|
||||
candidates: Vec<Candidate>,
|
||||
trial_progress: HashMap<u64, TrialProgress>,
|
||||
assigned_count: usize,
|
||||
generation_trial_ids: Vec<u64>,
|
||||
discovery_trial_id: Option<u64>,
|
||||
/// How many complete generations have been evaluated.
|
||||
generation: usize,
|
||||
}
|
||||
|
||||
impl Nsga2State {
|
||||
fn new(config: Nsga2Config, seed: Option<u64>) -> Self {
|
||||
let rng = seed.map_or_else(rand::make_rng, StdRng::seed_from_u64);
|
||||
Self {
|
||||
rng,
|
||||
config,
|
||||
phase: Phase::Discovery,
|
||||
dimensions: Vec::new(),
|
||||
population_size: 4,
|
||||
candidates: Vec::new(),
|
||||
trial_progress: HashMap::new(),
|
||||
assigned_count: 0,
|
||||
generation_trial_ids: Vec::new(),
|
||||
discovery_trial_id: None,
|
||||
generation: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MultiObjectiveSampler implementation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
impl crate::multi_objective::MultiObjectiveSampler for Nsga2Sampler {
|
||||
fn sample(
|
||||
&self,
|
||||
distribution: &Distribution,
|
||||
trial_id: u64,
|
||||
history: &[MultiObjectiveTrial],
|
||||
directions: &[Direction],
|
||||
) -> ParamValue {
|
||||
let mut state = self.state.lock();
|
||||
|
||||
match &state.phase {
|
||||
Phase::Discovery => sample_discovery(&mut state, distribution, trial_id),
|
||||
Phase::Active => sample_active(&mut state, distribution, trial_id, history, directions),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle sampling during the discovery phase.
|
||||
fn sample_discovery(
|
||||
state: &mut Nsga2State,
|
||||
distribution: &Distribution,
|
||||
trial_id: u64,
|
||||
) -> ParamValue {
|
||||
if let Some(prev_id) = state.discovery_trial_id
|
||||
&& trial_id != prev_id
|
||||
{
|
||||
finalize_discovery(state);
|
||||
// Assign this trial a random candidate (no history yet)
|
||||
generate_random_candidates(state);
|
||||
return sample_from_candidate(state, trial_id);
|
||||
}
|
||||
|
||||
state.discovery_trial_id = Some(trial_id);
|
||||
state.dimensions.push(DimensionInfo {
|
||||
distribution: distribution.clone(),
|
||||
});
|
||||
|
||||
sample_random(&mut state.rng, distribution)
|
||||
}
|
||||
|
||||
/// Transition from discovery to active phase.
|
||||
#[allow(
|
||||
clippy::cast_precision_loss,
|
||||
clippy::cast_possible_truncation,
|
||||
clippy::cast_sign_loss
|
||||
)]
|
||||
fn finalize_discovery(state: &mut Nsga2State) {
|
||||
let n = state.dimensions.len();
|
||||
state.population_size = state
|
||||
.config
|
||||
.user_population_size
|
||||
.unwrap_or_else(|| (4.0 + 3.0 * (n as f64).ln().max(0.0)).floor() as usize)
|
||||
.max(4);
|
||||
state.phase = Phase::Active;
|
||||
}
|
||||
|
||||
/// Generate `population_size` random candidates.
|
||||
fn generate_random_candidates(state: &mut Nsga2State) {
|
||||
let pop = state.population_size;
|
||||
state.candidates = (0..pop)
|
||||
.map(|_| {
|
||||
let params: Vec<ParamValue> = state
|
||||
.dimensions
|
||||
.iter()
|
||||
.map(|d| sample_random(&mut state.rng, &d.distribution))
|
||||
.collect();
|
||||
Candidate { params }
|
||||
})
|
||||
.collect();
|
||||
state.assigned_count = 0;
|
||||
state.generation_trial_ids.clear();
|
||||
state.trial_progress.clear();
|
||||
}
|
||||
|
||||
/// Active-phase sampling.
|
||||
fn sample_active(
|
||||
state: &mut Nsga2State,
|
||||
_distribution: &Distribution,
|
||||
trial_id: u64,
|
||||
history: &[MultiObjectiveTrial],
|
||||
directions: &[Direction],
|
||||
) -> ParamValue {
|
||||
// Check if we need to generate a new generation
|
||||
maybe_generate_new_generation(state, history, directions);
|
||||
|
||||
sample_from_candidate(state, trial_id)
|
||||
}
|
||||
|
||||
/// Assign a candidate to a trial and return the next dimension value.
|
||||
fn sample_from_candidate(state: &mut Nsga2State, trial_id: u64) -> ParamValue {
|
||||
// Assign candidate if not yet done
|
||||
if !state.trial_progress.contains_key(&trial_id) {
|
||||
let candidate_idx = if state.assigned_count < state.candidates.len() {
|
||||
let idx = state.assigned_count;
|
||||
state.assigned_count += 1;
|
||||
idx
|
||||
} else {
|
||||
// Overflow: generate a random candidate
|
||||
let params: Vec<ParamValue> = state
|
||||
.dimensions
|
||||
.iter()
|
||||
.map(|d| sample_random(&mut state.rng, &d.distribution))
|
||||
.collect();
|
||||
state.candidates.push(Candidate { params });
|
||||
let idx = state.candidates.len() - 1;
|
||||
state.assigned_count = state.candidates.len();
|
||||
idx
|
||||
};
|
||||
|
||||
state.trial_progress.insert(
|
||||
trial_id,
|
||||
TrialProgress {
|
||||
candidate_idx,
|
||||
next_dim: 0,
|
||||
},
|
||||
);
|
||||
state.generation_trial_ids.push(trial_id);
|
||||
}
|
||||
|
||||
let progress = state.trial_progress.get_mut(&trial_id).unwrap();
|
||||
let dim_idx = progress.next_dim;
|
||||
progress.next_dim += 1;
|
||||
|
||||
if dim_idx >= state.dimensions.len() {
|
||||
// Extra dimension: sample randomly
|
||||
return sample_random(
|
||||
&mut state.rng,
|
||||
&state.dimensions.last().unwrap().distribution,
|
||||
);
|
||||
}
|
||||
|
||||
state.candidates[progress.candidate_idx].params[dim_idx].clone()
|
||||
}
|
||||
|
||||
/// Check if all candidates in the current generation have been evaluated;
|
||||
/// if so, run NSGA-II selection and generate offspring.
|
||||
fn maybe_generate_new_generation(
|
||||
state: &mut Nsga2State,
|
||||
history: &[MultiObjectiveTrial],
|
||||
directions: &[Direction],
|
||||
) {
|
||||
let pop_size = state.population_size;
|
||||
|
||||
// Need at least pop_size assigned trials
|
||||
if state.generation_trial_ids.len() < pop_size {
|
||||
// Not enough candidates assigned yet — check if we need initial candidates
|
||||
if state.candidates.is_empty() {
|
||||
generate_random_candidates(state);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if the first pop_size trials are completed
|
||||
let gen_ids: Vec<u64> = state
|
||||
.generation_trial_ids
|
||||
.iter()
|
||||
.take(pop_size)
|
||||
.copied()
|
||||
.collect();
|
||||
let history_map: HashMap<u64, &MultiObjectiveTrial> =
|
||||
history.iter().map(|t| (t.id, t)).collect();
|
||||
|
||||
let all_completed = gen_ids.iter().all(|id| history_map.contains_key(id));
|
||||
if !all_completed {
|
||||
return;
|
||||
}
|
||||
|
||||
// Collect the evaluated population
|
||||
let evaluated: Vec<&MultiObjectiveTrial> = gen_ids
|
||||
.iter()
|
||||
.filter_map(|id| history_map.get(id).copied())
|
||||
.collect();
|
||||
|
||||
// Run NSGA-II to produce offspring
|
||||
let offspring = nsga2_generate_offspring(state, &evaluated, directions);
|
||||
state.candidates = offspring;
|
||||
state.assigned_count = 0;
|
||||
state.generation_trial_ids.clear();
|
||||
state.trial_progress.clear();
|
||||
state.generation += 1;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// NSGA-II generation algorithm
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Performs NSGA-II selection: non-dominated sort + crowding distance,
|
||||
/// then selects `pop_size` parents from the population.
|
||||
fn nsga2_select(
|
||||
state: &mut Nsga2State,
|
||||
population: &[&MultiObjectiveTrial],
|
||||
directions: &[Direction],
|
||||
) -> (Vec<Vec<ParamValue>>, Vec<usize>, Vec<f64>) {
|
||||
let pop_size = state.population_size;
|
||||
|
||||
let values: Vec<Vec<f64>> = population.iter().map(|t| t.values.clone()).collect();
|
||||
let constraints: Vec<Vec<f64>> = population.iter().map(|t| t.constraints.clone()).collect();
|
||||
let has_constraints = constraints.iter().any(|c| !c.is_empty());
|
||||
|
||||
let fronts = if has_constraints {
|
||||
pareto::fast_non_dominated_sort_constrained(&values, directions, &constraints)
|
||||
} else {
|
||||
pareto::fast_non_dominated_sort(&values, directions)
|
||||
};
|
||||
|
||||
let n = population.len();
|
||||
let mut rank = vec![0_usize; n];
|
||||
let mut crowding = vec![0.0_f64; n];
|
||||
|
||||
for (front_rank, front) in fronts.iter().enumerate() {
|
||||
let cd = pareto::crowding_distance_indexed(front, &values);
|
||||
for (i, &idx) in front.iter().enumerate() {
|
||||
rank[idx] = front_rank;
|
||||
crowding[idx] = cd[i];
|
||||
}
|
||||
}
|
||||
|
||||
let mut selected: Vec<usize> = Vec::with_capacity(pop_size);
|
||||
for front in &fronts {
|
||||
if selected.len() + front.len() <= pop_size {
|
||||
selected.extend_from_slice(front);
|
||||
} else {
|
||||
let remaining = pop_size - selected.len();
|
||||
let mut front_sorted: Vec<usize> = front.clone();
|
||||
front_sorted.sort_by(|&a, &b| {
|
||||
crowding[b]
|
||||
.partial_cmp(&crowding[a])
|
||||
.unwrap_or(core::cmp::Ordering::Equal)
|
||||
});
|
||||
selected.extend_from_slice(&front_sorted[..remaining]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
while selected.len() < pop_size {
|
||||
selected.push(state.rng.random_range(0..n));
|
||||
}
|
||||
|
||||
// Extract parent parameter vectors ordered by dimension
|
||||
let parents: Vec<Vec<ParamValue>> = selected
|
||||
.iter()
|
||||
.map(|&idx| extract_trial_params(population[idx], &state.dimensions, &mut state.rng))
|
||||
.collect();
|
||||
|
||||
let sel_rank: Vec<usize> = selected.iter().map(|&i| rank[i]).collect();
|
||||
let sel_crowding: Vec<f64> = selected.iter().map(|&i| crowding[i]).collect();
|
||||
|
||||
(parents, sel_rank, sel_crowding)
|
||||
}
|
||||
|
||||
/// Extract parameter values from a trial, ordered by dimension index.
|
||||
fn extract_trial_params(
|
||||
trial: &MultiObjectiveTrial,
|
||||
dimensions: &[DimensionInfo],
|
||||
rng: &mut StdRng,
|
||||
) -> Vec<ParamValue> {
|
||||
let mut param_pairs: Vec<_> = trial.params.iter().collect();
|
||||
param_pairs.sort_by_key(|(id, _)| *id);
|
||||
|
||||
dimensions
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(dim_idx, dim_info)| {
|
||||
if dim_idx < param_pairs.len() {
|
||||
param_pairs[dim_idx].1.clone()
|
||||
} else {
|
||||
sample_random(rng, &dim_info.distribution)
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Runs NSGA-II selection and generates offspring candidates.
|
||||
fn nsga2_generate_offspring(
|
||||
state: &mut Nsga2State,
|
||||
population: &[&MultiObjectiveTrial],
|
||||
directions: &[Direction],
|
||||
) -> Vec<Candidate> {
|
||||
let pop_size = state.population_size;
|
||||
|
||||
if population.len() < 2 {
|
||||
return (0..pop_size)
|
||||
.map(|_| {
|
||||
let params = state
|
||||
.dimensions
|
||||
.iter()
|
||||
.map(|d| sample_random(&mut state.rng, &d.distribution))
|
||||
.collect();
|
||||
Candidate { params }
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
|
||||
let (parents, sel_rank, sel_crowding) = nsga2_select(state, population, directions);
|
||||
|
||||
let mut offspring = Vec::with_capacity(pop_size);
|
||||
while offspring.len() < pop_size {
|
||||
let p1 = tournament_select(&mut state.rng, &sel_rank, &sel_crowding, parents.len());
|
||||
let p2 = tournament_select(&mut state.rng, &sel_rank, &sel_crowding, parents.len());
|
||||
|
||||
let (mut child1, mut child2) = crossover(
|
||||
&mut state.rng,
|
||||
&parents[p1],
|
||||
&parents[p2],
|
||||
&state.dimensions,
|
||||
state.config.crossover_prob,
|
||||
state.config.crossover_eta,
|
||||
);
|
||||
|
||||
mutate(
|
||||
&mut state.rng,
|
||||
&mut child1,
|
||||
&state.dimensions,
|
||||
state.config.mutation_eta,
|
||||
);
|
||||
mutate(
|
||||
&mut state.rng,
|
||||
&mut child2,
|
||||
&state.dimensions,
|
||||
state.config.mutation_eta,
|
||||
);
|
||||
|
||||
offspring.push(Candidate { params: child1 });
|
||||
if offspring.len() < pop_size {
|
||||
offspring.push(Candidate { params: child2 });
|
||||
}
|
||||
}
|
||||
|
||||
offspring
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Genetic operators
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Tournament selection: pick 2 random individuals, return index of winner.
|
||||
/// Winner has lower rank; ties broken by higher crowding distance.
|
||||
fn tournament_select(rng: &mut StdRng, ranks: &[usize], crowding: &[f64], n: usize) -> usize {
|
||||
let a = rng.random_range(0..n);
|
||||
let b = rng.random_range(0..n);
|
||||
|
||||
if ranks[a] < ranks[b] {
|
||||
a
|
||||
} else if ranks[b] < ranks[a] {
|
||||
b
|
||||
} else if crowding[a] >= crowding[b] {
|
||||
a
|
||||
} else {
|
||||
b
|
||||
}
|
||||
}
|
||||
|
||||
/// SBX crossover for continuous params, uniform crossover for categorical.
|
||||
fn crossover(
|
||||
rng: &mut StdRng,
|
||||
parent1: &[ParamValue],
|
||||
parent2: &[ParamValue],
|
||||
dimensions: &[DimensionInfo],
|
||||
crossover_prob: f64,
|
||||
eta: f64,
|
||||
) -> (Vec<ParamValue>, Vec<ParamValue>) {
|
||||
let n = parent1.len();
|
||||
let mut child1 = parent1.to_vec();
|
||||
let mut child2 = parent2.to_vec();
|
||||
|
||||
let u: f64 = rng.random_range(0.0..=1.0);
|
||||
if u > crossover_prob {
|
||||
return (child1, child2);
|
||||
}
|
||||
|
||||
for i in 0..n {
|
||||
match (&parent1[i], &parent2[i], &dimensions[i].distribution) {
|
||||
(ParamValue::Float(p1), ParamValue::Float(p2), Distribution::Float(d)) => {
|
||||
if (p1 - p2).abs() < 1e-14 {
|
||||
continue;
|
||||
}
|
||||
let (c1, c2) = sbx_crossover_f64(rng, *p1, *p2, d.low, d.high, eta);
|
||||
child1[i] = ParamValue::Float(c1);
|
||||
child2[i] = ParamValue::Float(c2);
|
||||
}
|
||||
(ParamValue::Int(p1), ParamValue::Int(p2), Distribution::Int(d)) => {
|
||||
if p1 == p2 {
|
||||
continue;
|
||||
}
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
let (c1, c2) = sbx_crossover_f64(
|
||||
rng,
|
||||
*p1 as f64,
|
||||
*p2 as f64,
|
||||
d.low as f64,
|
||||
d.high as f64,
|
||||
eta,
|
||||
);
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
{
|
||||
child1[i] = ParamValue::Int((c1.round() as i64).clamp(d.low, d.high));
|
||||
child2[i] = ParamValue::Int((c2.round() as i64).clamp(d.low, d.high));
|
||||
}
|
||||
}
|
||||
(ParamValue::Categorical(_), ParamValue::Categorical(_), _) => {
|
||||
// Uniform crossover: swap with 50% probability
|
||||
if rng.random_range(0.0..=1.0) < 0.5 {
|
||||
core::mem::swap(&mut child1[i], &mut child2[i]);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
(child1, child2)
|
||||
}
|
||||
|
||||
/// SBX crossover for a single float dimension.
|
||||
fn sbx_crossover_f64(
|
||||
rng: &mut StdRng,
|
||||
p1: f64,
|
||||
p2: f64,
|
||||
low: f64,
|
||||
high: f64,
|
||||
eta: f64,
|
||||
) -> (f64, f64) {
|
||||
let u: f64 = rng.random_range(0.0_f64..1.0_f64);
|
||||
|
||||
let beta = if u <= 0.5 {
|
||||
(2.0 * u).powf(1.0 / (eta + 1.0))
|
||||
} else {
|
||||
(1.0 / (2.0 * (1.0 - u))).powf(1.0 / (eta + 1.0))
|
||||
};
|
||||
|
||||
let c1 = 0.5 * ((1.0 + beta) * p1 + (1.0 - beta) * p2);
|
||||
let c2 = 0.5 * ((1.0 - beta) * p1 + (1.0 + beta) * p2);
|
||||
|
||||
(c1.clamp(low, high), c2.clamp(low, high))
|
||||
}
|
||||
|
||||
/// Polynomial mutation for each dimension.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
fn mutate(rng: &mut StdRng, individual: &mut [ParamValue], dimensions: &[DimensionInfo], eta: f64) {
|
||||
let n = individual.len();
|
||||
if n == 0 {
|
||||
return;
|
||||
}
|
||||
let mutation_prob = 1.0 / n as f64;
|
||||
|
||||
for (i, value) in individual.iter_mut().enumerate() {
|
||||
if rng.random_range(0.0..=1.0) >= mutation_prob {
|
||||
continue;
|
||||
}
|
||||
|
||||
match (value, &dimensions[i].distribution) {
|
||||
(v @ ParamValue::Float(_), Distribution::Float(d)) => {
|
||||
let ParamValue::Float(x) = *v else {
|
||||
unreachable!();
|
||||
};
|
||||
let mutated = polynomial_mutation_f64(rng, x, d.low, d.high, eta);
|
||||
*v = ParamValue::Float(mutated);
|
||||
}
|
||||
(v @ ParamValue::Int(_), Distribution::Int(d)) => {
|
||||
let ParamValue::Int(x) = *v else {
|
||||
unreachable!();
|
||||
};
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
{
|
||||
let mutated =
|
||||
polynomial_mutation_f64(rng, x as f64, d.low as f64, d.high as f64, eta);
|
||||
*v = ParamValue::Int((mutated.round() as i64).clamp(d.low, d.high));
|
||||
}
|
||||
}
|
||||
(v @ ParamValue::Categorical(_), Distribution::Categorical(d)) => {
|
||||
*v = ParamValue::Categorical(rng.random_range(0..d.n_choices));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Polynomial mutation for a single float value.
|
||||
fn polynomial_mutation_f64(rng: &mut StdRng, x: f64, low: f64, high: f64, eta: f64) -> f64 {
|
||||
let u: f64 = rng.random_range(0.0_f64..1.0_f64);
|
||||
let range = high - low;
|
||||
if range <= 0.0 {
|
||||
return x;
|
||||
}
|
||||
|
||||
let delta1 = (x - low) / range;
|
||||
let delta2 = (high - x) / range;
|
||||
|
||||
let delta_q = if u < 0.5 {
|
||||
let xy = 1.0 - delta1;
|
||||
let val = 2.0 * u + (1.0 - 2.0 * u) * xy.powf(eta + 1.0);
|
||||
val.powf(1.0 / (eta + 1.0)) - 1.0
|
||||
} else {
|
||||
let xy = 1.0 - delta2;
|
||||
let val = 2.0 * (1.0 - u) + 2.0 * (u - 0.5) * xy.powf(eta + 1.0);
|
||||
1.0 - val.powf(1.0 / (eta + 1.0))
|
||||
};
|
||||
|
||||
(x + delta_q * range).clamp(low, high)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Random sampling helper (for discovery phase)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[allow(clippy::cast_possible_truncation, clippy::cast_precision_loss)]
|
||||
fn sample_random(rng: &mut StdRng, distribution: &Distribution) -> ParamValue {
|
||||
match distribution {
|
||||
Distribution::Float(d) => {
|
||||
let value = if d.log_scale {
|
||||
let log_low = d.low.ln();
|
||||
let log_high = d.high.ln();
|
||||
rng.random_range(log_low..=log_high).exp()
|
||||
} else if let Some(step) = d.step {
|
||||
let n_steps = ((d.high - d.low) / step).floor() as i64;
|
||||
let k = rng.random_range(0..=n_steps);
|
||||
d.low + (k as f64) * step
|
||||
} else {
|
||||
rng.random_range(d.low..=d.high)
|
||||
};
|
||||
ParamValue::Float(value)
|
||||
}
|
||||
Distribution::Int(d) => {
|
||||
let value = if d.log_scale {
|
||||
let log_low = (d.low as f64).ln();
|
||||
let log_high = (d.high as f64).ln();
|
||||
let raw = rng.random_range(log_low..=log_high).exp().round() as i64;
|
||||
raw.clamp(d.low, d.high)
|
||||
} else if let Some(step) = d.step {
|
||||
let n_steps = (d.high - d.low) / step;
|
||||
let k = rng.random_range(0..=n_steps);
|
||||
d.low + k * step
|
||||
} else {
|
||||
rng.random_range(d.low..=d.high)
|
||||
};
|
||||
ParamValue::Int(value)
|
||||
}
|
||||
Distribution::Categorical(d) => ParamValue::Categorical(rng.random_range(0..d.n_choices)),
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use rand::rngs::StdRng;
|
||||
use rand::{RngExt, SeedableRng};
|
||||
use rand::{Rng, SeedableRng};
|
||||
|
||||
use crate::distribution::Distribution;
|
||||
use crate::param::ParamValue;
|
||||
@@ -17,7 +17,7 @@ use crate::sampler::{CompletedTrial, Sampler};
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::sampler::random::RandomSampler;
|
||||
/// use optimizer::RandomSampler;
|
||||
///
|
||||
/// // Create with default RNG
|
||||
/// let sampler = RandomSampler::new();
|
||||
@@ -31,17 +31,15 @@ pub struct RandomSampler {
|
||||
|
||||
impl RandomSampler {
|
||||
/// Creates a new random sampler with a default random seed.
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
rng: Mutex::new(rand::make_rng()),
|
||||
rng: Mutex::new(StdRng::from_os_rng()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a new random sampler with a fixed seed for reproducibility.
|
||||
///
|
||||
/// Using the same seed will produce the same sequence of sampled values.
|
||||
#[must_use]
|
||||
pub fn with_seed(seed: u64) -> Self {
|
||||
Self {
|
||||
rng: Mutex::new(StdRng::seed_from_u64(seed)),
|
||||
@@ -56,7 +54,6 @@ impl Default for RandomSampler {
|
||||
}
|
||||
|
||||
impl Sampler for RandomSampler {
|
||||
#[allow(clippy::cast_possible_truncation, clippy::cast_precision_loss)]
|
||||
fn sample(
|
||||
&self,
|
||||
distribution: &Distribution,
|
||||
@@ -113,7 +110,6 @@ impl Sampler for RandomSampler {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::cast_possible_truncation, clippy::cast_precision_loss)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::distribution::{CategoricalDistribution, FloatDistribution, IntDistribution};
|
||||
|
||||
@@ -1,420 +0,0 @@
|
||||
//! Quasi-random sampler using Sobol low-discrepancy sequences.
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use sobol_burley::sample;
|
||||
|
||||
use crate::distribution::Distribution;
|
||||
use crate::param::ParamValue;
|
||||
use crate::sampler::{CompletedTrial, Sampler};
|
||||
|
||||
/// Internal state for tracking the dimension counter within a trial.
|
||||
struct SobolState {
|
||||
/// The `trial_id` of the current trial (used to reset dimension counter).
|
||||
current_trial: u64,
|
||||
/// Next Sobol dimension to use for the current trial.
|
||||
next_dimension: u32,
|
||||
}
|
||||
|
||||
/// Quasi-random sampler using Sobol low-discrepancy sequences.
|
||||
///
|
||||
/// Provides better uniform coverage of the parameter space than
|
||||
/// [`RandomSampler`](super::random::RandomSampler). Useful as a baseline or
|
||||
/// for the startup phase of model-based samplers.
|
||||
///
|
||||
/// Unlike random sampling, Sobol sequences are deterministic and fill the
|
||||
/// space more evenly, reducing the number of trials needed to adequately
|
||||
/// cover the search space.
|
||||
///
|
||||
/// Each trial uses a different Sobol sequence index, and each parameter
|
||||
/// within a trial maps to a different Sobol dimension. Parameters must be
|
||||
/// suggested in the same order across trials for consistent dimension
|
||||
/// assignment.
|
||||
///
|
||||
/// Sobol sequences are most effective in moderate dimensions (up to ~20).
|
||||
/// For very high dimensions, the uniformity advantage diminishes.
|
||||
///
|
||||
/// Requires the `sobol` feature flag.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::sampler::sobol::SobolSampler;
|
||||
/// use optimizer::{Direction, Study};
|
||||
///
|
||||
/// let study: Study<f64> = Study::with_sampler(Direction::Minimize, SobolSampler::new());
|
||||
/// ```
|
||||
pub struct SobolSampler {
|
||||
seed: u32,
|
||||
state: Mutex<SobolState>,
|
||||
}
|
||||
|
||||
impl SobolSampler {
|
||||
/// Creates a new Sobol sampler with a default seed of 0.
|
||||
#[must_use]
|
||||
pub fn new() -> Self {
|
||||
Self::with_seed(0)
|
||||
}
|
||||
|
||||
/// Creates a new Sobol sampler with the given seed.
|
||||
///
|
||||
/// Different seeds produce statistically independent Sobol sequences.
|
||||
/// Using the same seed will produce the same sequence of sampled values.
|
||||
#[must_use]
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
pub fn with_seed(seed: u64) -> Self {
|
||||
Self {
|
||||
seed: seed as u32,
|
||||
state: Mutex::new(SobolState {
|
||||
current_trial: u64::MAX,
|
||||
next_dimension: 0,
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SobolSampler {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl Sampler for SobolSampler {
|
||||
#[allow(clippy::cast_possible_truncation, clippy::cast_precision_loss)]
|
||||
fn sample(
|
||||
&self,
|
||||
distribution: &Distribution,
|
||||
trial_id: u64,
|
||||
_history: &[CompletedTrial],
|
||||
) -> ParamValue {
|
||||
let mut state = self.state.lock();
|
||||
|
||||
// Reset dimension counter when a new trial starts.
|
||||
if state.current_trial != trial_id {
|
||||
state.current_trial = trial_id;
|
||||
state.next_dimension = 0;
|
||||
}
|
||||
|
||||
let dimension = state.next_dimension;
|
||||
state.next_dimension = dimension + 1;
|
||||
|
||||
// Use trial_id as the Sobol sequence index.
|
||||
let index = trial_id as u32;
|
||||
|
||||
// Generate a quasi-random point in [0, 1).
|
||||
let point = f64::from(sample(index, dimension, self.seed));
|
||||
|
||||
map_point_to_distribution(point, distribution)
|
||||
}
|
||||
}
|
||||
|
||||
/// Maps a uniform [0, 1) point to a value within the given distribution.
|
||||
#[allow(
|
||||
clippy::cast_possible_truncation,
|
||||
clippy::cast_precision_loss,
|
||||
clippy::cast_sign_loss
|
||||
)]
|
||||
fn map_point_to_distribution(point: f64, distribution: &Distribution) -> ParamValue {
|
||||
match distribution {
|
||||
Distribution::Float(d) => {
|
||||
let value = if d.log_scale {
|
||||
let log_low = d.low.ln();
|
||||
let log_high = d.high.ln();
|
||||
(log_low + point * (log_high - log_low)).exp()
|
||||
} else if let Some(step) = d.step {
|
||||
let n_steps = ((d.high - d.low) / step).floor() as i64;
|
||||
let k = (point * (n_steps + 1) as f64).floor() as i64;
|
||||
let k = k.min(n_steps);
|
||||
d.low + (k as f64) * step
|
||||
} else {
|
||||
d.low + point * (d.high - d.low)
|
||||
};
|
||||
ParamValue::Float(value)
|
||||
}
|
||||
Distribution::Int(d) => {
|
||||
let value = if d.log_scale {
|
||||
let log_low = (d.low as f64).ln();
|
||||
let log_high = (d.high as f64).ln();
|
||||
let raw = (log_low + point * (log_high - log_low)).exp().round() as i64;
|
||||
raw.clamp(d.low, d.high)
|
||||
} else if let Some(step) = d.step {
|
||||
let n_steps = (d.high - d.low) / step;
|
||||
let k = (point * (n_steps + 1) as f64).floor() as i64;
|
||||
let k = k.min(n_steps);
|
||||
d.low + k * step
|
||||
} else {
|
||||
let range = d.high - d.low + 1;
|
||||
let k = (point * range as f64).floor() as i64;
|
||||
(d.low + k).min(d.high)
|
||||
};
|
||||
ParamValue::Int(value)
|
||||
}
|
||||
Distribution::Categorical(d) => {
|
||||
let index = (point * d.n_choices as f64).floor() as usize;
|
||||
let index = index.min(d.n_choices - 1);
|
||||
ParamValue::Categorical(index)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(
|
||||
clippy::cast_possible_truncation,
|
||||
clippy::cast_precision_loss,
|
||||
clippy::cast_sign_loss
|
||||
)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::distribution::{CategoricalDistribution, FloatDistribution, IntDistribution};
|
||||
|
||||
#[test]
|
||||
fn float_within_bounds() {
|
||||
let sampler = SobolSampler::with_seed(42);
|
||||
let dist = Distribution::Float(FloatDistribution {
|
||||
low: -5.0,
|
||||
high: 5.0,
|
||||
log_scale: false,
|
||||
step: None,
|
||||
});
|
||||
|
||||
for i in 0..100 {
|
||||
let value = sampler.sample(&dist, i, &[]);
|
||||
if let ParamValue::Float(v) = value {
|
||||
assert!(
|
||||
(-5.0..=5.0).contains(&v),
|
||||
"value {v} out of bounds at trial {i}"
|
||||
);
|
||||
} else {
|
||||
panic!("Expected Float value");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn float_log_scale_within_bounds() {
|
||||
let sampler = SobolSampler::with_seed(42);
|
||||
let dist = Distribution::Float(FloatDistribution {
|
||||
low: 1e-5,
|
||||
high: 1.0,
|
||||
log_scale: true,
|
||||
step: None,
|
||||
});
|
||||
|
||||
for i in 0..100 {
|
||||
let value = sampler.sample(&dist, i, &[]);
|
||||
if let ParamValue::Float(v) = value {
|
||||
assert!(
|
||||
(1e-5..=1.0).contains(&v),
|
||||
"value {v} out of bounds at trial {i}"
|
||||
);
|
||||
} else {
|
||||
panic!("Expected Float value");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn float_step_respects_grid() {
|
||||
let sampler = SobolSampler::with_seed(42);
|
||||
let dist = Distribution::Float(FloatDistribution {
|
||||
low: 0.0,
|
||||
high: 1.0,
|
||||
log_scale: false,
|
||||
step: Some(0.25),
|
||||
});
|
||||
|
||||
for i in 0..100 {
|
||||
let value = sampler.sample(&dist, i, &[]);
|
||||
if let ParamValue::Float(v) = value {
|
||||
assert!((0.0..=1.0).contains(&v), "value {v} out of bounds");
|
||||
let k = (v / 0.25).round() as i64;
|
||||
let expected = k as f64 * 0.25;
|
||||
assert!((v - expected).abs() < 1e-10, "value {v} not on step grid");
|
||||
} else {
|
||||
panic!("Expected Float value");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn int_within_bounds() {
|
||||
let sampler = SobolSampler::with_seed(42);
|
||||
let dist = Distribution::Int(IntDistribution {
|
||||
low: 0,
|
||||
high: 10,
|
||||
log_scale: false,
|
||||
step: None,
|
||||
});
|
||||
|
||||
for i in 0..100 {
|
||||
let value = sampler.sample(&dist, i, &[]);
|
||||
if let ParamValue::Int(v) = value {
|
||||
assert!(
|
||||
(0..=10).contains(&v),
|
||||
"value {v} out of bounds at trial {i}"
|
||||
);
|
||||
} else {
|
||||
panic!("Expected Int value");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn int_log_scale_within_bounds() {
|
||||
let sampler = SobolSampler::with_seed(42);
|
||||
let dist = Distribution::Int(IntDistribution {
|
||||
low: 1,
|
||||
high: 1000,
|
||||
log_scale: true,
|
||||
step: None,
|
||||
});
|
||||
|
||||
for i in 0..100 {
|
||||
let value = sampler.sample(&dist, i, &[]);
|
||||
if let ParamValue::Int(v) = value {
|
||||
assert!(
|
||||
(1..=1000).contains(&v),
|
||||
"value {v} out of bounds at trial {i}"
|
||||
);
|
||||
} else {
|
||||
panic!("Expected Int value");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn int_step_respects_grid() {
|
||||
let sampler = SobolSampler::with_seed(42);
|
||||
let dist = Distribution::Int(IntDistribution {
|
||||
low: 0,
|
||||
high: 10,
|
||||
log_scale: false,
|
||||
step: Some(2),
|
||||
});
|
||||
|
||||
for i in 0..100 {
|
||||
let value = sampler.sample(&dist, i, &[]);
|
||||
if let ParamValue::Int(v) = value {
|
||||
assert!((0..=10).contains(&v), "value {v} out of bounds");
|
||||
assert!(v % 2 == 0, "value {v} not on step grid");
|
||||
} else {
|
||||
panic!("Expected Int value");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn categorical_within_bounds() {
|
||||
let sampler = SobolSampler::with_seed(42);
|
||||
let dist = Distribution::Categorical(CategoricalDistribution { n_choices: 5 });
|
||||
|
||||
for i in 0..100 {
|
||||
let value = sampler.sample(&dist, i, &[]);
|
||||
if let ParamValue::Categorical(idx) = value {
|
||||
assert!(idx < 5, "index {idx} out of bounds at trial {i}");
|
||||
} else {
|
||||
panic!("Expected Categorical value");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deterministic_with_same_seed() {
|
||||
let dist = Distribution::Float(FloatDistribution {
|
||||
low: 0.0,
|
||||
high: 1.0,
|
||||
log_scale: false,
|
||||
step: None,
|
||||
});
|
||||
|
||||
let sampler1 = SobolSampler::with_seed(42);
|
||||
let sampler2 = SobolSampler::with_seed(42);
|
||||
|
||||
for i in 0..20 {
|
||||
let v1 = sampler1.sample(&dist, i, &[]);
|
||||
let v2 = sampler2.sample(&dist, i, &[]);
|
||||
assert_eq!(v1, v2, "mismatch at trial {i}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn different_seeds_produce_different_sequences() {
|
||||
let dist = Distribution::Float(FloatDistribution {
|
||||
low: 0.0,
|
||||
high: 1.0,
|
||||
log_scale: false,
|
||||
step: None,
|
||||
});
|
||||
|
||||
let sampler1 = SobolSampler::with_seed(0);
|
||||
let sampler2 = SobolSampler::with_seed(12345);
|
||||
|
||||
let mut any_different = false;
|
||||
for i in 0..20 {
|
||||
let v1 = sampler1.sample(&dist, i, &[]);
|
||||
let v2 = sampler2.sample(&dist, i, &[]);
|
||||
if v1 != v2 {
|
||||
any_different = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
any_different,
|
||||
"different seeds should produce different sequences"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn better_coverage_than_random() {
|
||||
// Sobol sequence should cover [0,1] more uniformly than random.
|
||||
// We measure this by checking that the Sobol samples fill all
|
||||
// 10 equal-width bins with only 20 samples.
|
||||
let sampler = SobolSampler::with_seed(0);
|
||||
let dist = Distribution::Float(FloatDistribution {
|
||||
low: 0.0,
|
||||
high: 1.0,
|
||||
log_scale: false,
|
||||
step: None,
|
||||
});
|
||||
|
||||
let n_bins = 10;
|
||||
let n_samples = 20;
|
||||
let mut bins = vec![0u32; n_bins];
|
||||
|
||||
for i in 0..n_samples {
|
||||
let value = sampler.sample(&dist, i, &[]);
|
||||
if let ParamValue::Float(v) = value {
|
||||
let bin = ((v * n_bins as f64).floor() as usize).min(n_bins - 1);
|
||||
bins[bin] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
let filled_bins = bins.iter().filter(|&&c| c > 0).count();
|
||||
assert!(
|
||||
filled_bins >= 8,
|
||||
"Expected at least 8/10 bins filled, got {filled_bins}: {bins:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multi_parameter_uses_different_dimensions() {
|
||||
// When sampling multiple parameters per trial, each parameter
|
||||
// should get a different Sobol dimension, producing different values.
|
||||
let sampler = SobolSampler::with_seed(0);
|
||||
let dist = Distribution::Float(FloatDistribution {
|
||||
low: 0.0,
|
||||
high: 1.0,
|
||||
log_scale: false,
|
||||
step: None,
|
||||
});
|
||||
|
||||
// Sample two parameters for trial 0.
|
||||
let v1 = sampler.sample(&dist, 0, &[]);
|
||||
let v2 = sampler.sample(&dist, 0, &[]);
|
||||
|
||||
// They should differ (different Sobol dimensions for the same index).
|
||||
assert_ne!(
|
||||
v1, v2,
|
||||
"multi-parameter samples should use different dimensions"
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,670 +0,0 @@
|
||||
use core::fmt::Debug;
|
||||
|
||||
use crate::Error;
|
||||
|
||||
/// A strategy for computing the gamma quantile in TPE.
|
||||
///
|
||||
/// The gamma value determines what fraction of trials are considered "good"
|
||||
/// when splitting the trial history. Different strategies can adapt this
|
||||
/// fraction based on the number of completed trials.
|
||||
///
|
||||
/// # Implementation Notes
|
||||
///
|
||||
/// - The returned gamma must be in the range (0.0, 1.0)
|
||||
/// - Implementations should be deterministic for reproducibility
|
||||
/// - The `clone_box` method enables trait object cloning
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::sampler::tpe::GammaStrategy;
|
||||
///
|
||||
/// #[derive(Debug, Clone)]
|
||||
/// struct ConstantGamma(f64);
|
||||
///
|
||||
/// impl GammaStrategy for ConstantGamma {
|
||||
/// fn gamma(&self, _n_trials: usize) -> f64 {
|
||||
/// self.0
|
||||
/// }
|
||||
///
|
||||
/// fn clone_box(&self) -> Box<dyn GammaStrategy> {
|
||||
/// Box::new(self.clone())
|
||||
/// }
|
||||
/// }
|
||||
/// ```
|
||||
pub trait GammaStrategy: Send + Sync + Debug {
|
||||
/// Computes the gamma quantile based on the number of completed trials.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `n_trials` - The number of completed trials in the history.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// A gamma value in the range (0.0, 1.0). Values outside this range
|
||||
/// will be clamped by the sampler.
|
||||
fn gamma(&self, n_trials: usize) -> f64;
|
||||
|
||||
/// Creates a boxed clone of this strategy.
|
||||
///
|
||||
/// This method enables cloning of trait objects, which is necessary
|
||||
/// for the builder pattern and sampler configuration.
|
||||
fn clone_box(&self) -> Box<dyn GammaStrategy>;
|
||||
}
|
||||
|
||||
impl Clone for Box<dyn GammaStrategy> {
|
||||
fn clone(&self) -> Self {
|
||||
self.clone_box()
|
||||
}
|
||||
}
|
||||
|
||||
/// A fixed gamma strategy that returns a constant value.
|
||||
///
|
||||
/// This is the simplest strategy and the default behavior of TPE.
|
||||
/// The gamma value remains constant regardless of the number of trials.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::sampler::tpe::{FixedGamma, TpeSampler};
|
||||
///
|
||||
/// // Use 15% of trials as "good"
|
||||
/// let sampler = TpeSampler::builder()
|
||||
/// .gamma_strategy(FixedGamma::new(0.15).unwrap())
|
||||
/// .build()
|
||||
/// .unwrap();
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct FixedGamma {
|
||||
gamma: f64,
|
||||
}
|
||||
|
||||
impl FixedGamma {
|
||||
/// Creates a new fixed gamma strategy.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `gamma` - The constant gamma value to use.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns `Error::InvalidGamma` if gamma is not in (0.0, 1.0).
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::sampler::tpe::FixedGamma;
|
||||
///
|
||||
/// let strategy = FixedGamma::new(0.25).unwrap();
|
||||
/// assert!((strategy.value() - 0.25).abs() < f64::EPSILON);
|
||||
/// ```
|
||||
pub fn new(gamma: f64) -> crate::Result<Self> {
|
||||
if gamma <= 0.0 || gamma >= 1.0 {
|
||||
return Err(Error::InvalidGamma(gamma));
|
||||
}
|
||||
Ok(Self { gamma })
|
||||
}
|
||||
|
||||
/// Returns the fixed gamma value.
|
||||
#[must_use]
|
||||
pub fn value(&self) -> f64 {
|
||||
self.gamma
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for FixedGamma {
|
||||
/// Creates a fixed gamma strategy with the default value of 0.25.
|
||||
fn default() -> Self {
|
||||
Self { gamma: 0.25 }
|
||||
}
|
||||
}
|
||||
|
||||
impl GammaStrategy for FixedGamma {
|
||||
fn gamma(&self, _n_trials: usize) -> f64 {
|
||||
self.gamma
|
||||
}
|
||||
|
||||
fn clone_box(&self) -> Box<dyn GammaStrategy> {
|
||||
Box::new(*self)
|
||||
}
|
||||
}
|
||||
|
||||
/// A linear gamma strategy that interpolates between min and max values.
|
||||
///
|
||||
/// The gamma value increases linearly from `gamma_min` to `gamma_max` as the
|
||||
/// number of trials grows from 0 to `n_trials_max`. Beyond `n_trials_max`,
|
||||
/// gamma remains at `gamma_max`.
|
||||
///
|
||||
/// This strategy is useful when you want to be more explorative early on
|
||||
/// (smaller gamma = fewer "good" trials) and more exploitative later
|
||||
/// (larger gamma = more "good" trials).
|
||||
///
|
||||
/// # Formula
|
||||
///
|
||||
/// ```text
|
||||
/// gamma = gamma_min + (gamma_max - gamma_min) * min(n_trials / n_trials_max, 1.0)
|
||||
/// ```
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::sampler::tpe::{GammaStrategy, LinearGamma, TpeSampler};
|
||||
///
|
||||
/// let strategy = LinearGamma::new(0.1, 0.4, 100).unwrap();
|
||||
///
|
||||
/// // At 0 trials: gamma = 0.1
|
||||
/// assert!((strategy.gamma(0) - 0.1).abs() < f64::EPSILON);
|
||||
///
|
||||
/// // At 50 trials: gamma = 0.25 (midpoint)
|
||||
/// assert!((strategy.gamma(50) - 0.25).abs() < f64::EPSILON);
|
||||
///
|
||||
/// // At 100+ trials: gamma = 0.4
|
||||
/// assert!((strategy.gamma(100) - 0.4).abs() < f64::EPSILON);
|
||||
/// assert!((strategy.gamma(200) - 0.4).abs() < f64::EPSILON);
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct LinearGamma {
|
||||
gamma_min: f64,
|
||||
gamma_max: f64,
|
||||
n_trials_max: usize,
|
||||
}
|
||||
|
||||
impl LinearGamma {
|
||||
/// Creates a new linear gamma strategy.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `gamma_min` - The minimum gamma value (at 0 trials).
|
||||
/// * `gamma_max` - The maximum gamma value (at `n_trials_max` trials).
|
||||
/// * `n_trials_max` - The number of trials at which gamma reaches its maximum.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns `Error::InvalidGamma` if:
|
||||
/// - `gamma_min` is not in (0.0, 1.0)
|
||||
/// - `gamma_max` is not in (0.0, 1.0)
|
||||
/// - `gamma_min > gamma_max`
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::sampler::tpe::LinearGamma;
|
||||
///
|
||||
/// // Gamma goes from 0.1 to 0.3 over 50 trials
|
||||
/// let strategy = LinearGamma::new(0.1, 0.3, 50).unwrap();
|
||||
/// ```
|
||||
pub fn new(gamma_min: f64, gamma_max: f64, n_trials_max: usize) -> crate::Result<Self> {
|
||||
if gamma_min <= 0.0 || gamma_min >= 1.0 {
|
||||
return Err(Error::InvalidGamma(gamma_min));
|
||||
}
|
||||
if gamma_max <= 0.0 || gamma_max >= 1.0 {
|
||||
return Err(Error::InvalidGamma(gamma_max));
|
||||
}
|
||||
if gamma_min > gamma_max {
|
||||
return Err(Error::InvalidGamma(gamma_min));
|
||||
}
|
||||
Ok(Self {
|
||||
gamma_min,
|
||||
gamma_max,
|
||||
n_trials_max,
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the minimum gamma value.
|
||||
#[must_use]
|
||||
pub fn gamma_min(&self) -> f64 {
|
||||
self.gamma_min
|
||||
}
|
||||
|
||||
/// Returns the maximum gamma value.
|
||||
#[must_use]
|
||||
pub fn gamma_max(&self) -> f64 {
|
||||
self.gamma_max
|
||||
}
|
||||
|
||||
/// Returns the number of trials at which gamma reaches its maximum.
|
||||
#[must_use]
|
||||
pub fn n_trials_max(&self) -> usize {
|
||||
self.n_trials_max
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for LinearGamma {
|
||||
/// Creates a linear gamma strategy with default values:
|
||||
/// - `gamma_min`: 0.10
|
||||
/// - `gamma_max`: 0.25
|
||||
/// - `n_trials_max`: 100
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
gamma_min: 0.10,
|
||||
gamma_max: 0.25,
|
||||
n_trials_max: 100,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl GammaStrategy for LinearGamma {
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
fn gamma(&self, n_trials: usize) -> f64 {
|
||||
if self.n_trials_max == 0 {
|
||||
return self.gamma_max;
|
||||
}
|
||||
let t = (n_trials as f64 / self.n_trials_max as f64).min(1.0);
|
||||
self.gamma_min + (self.gamma_max - self.gamma_min) * t
|
||||
}
|
||||
|
||||
fn clone_box(&self) -> Box<dyn GammaStrategy> {
|
||||
Box::new(*self)
|
||||
}
|
||||
}
|
||||
|
||||
/// A square root gamma strategy inspired by Optuna's default behavior.
|
||||
///
|
||||
/// The gamma value is computed based on the inverse square root of the number
|
||||
/// of trials, providing a balance between exploration and exploitation that
|
||||
/// naturally adapts as more data becomes available.
|
||||
///
|
||||
/// # Formula
|
||||
///
|
||||
/// ```text
|
||||
/// n_good = max(1, floor(gamma_factor / sqrt(n_trials)))
|
||||
/// gamma = min(gamma_max, n_good / n_trials)
|
||||
/// ```
|
||||
///
|
||||
/// When `n_trials` is 0, returns `gamma_max`.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::sampler::tpe::{GammaStrategy, SqrtGamma, TpeSampler};
|
||||
///
|
||||
/// let strategy = SqrtGamma::default();
|
||||
///
|
||||
/// // Gamma decreases as trials increase
|
||||
/// let g10 = strategy.gamma(10);
|
||||
/// let g100 = strategy.gamma(100);
|
||||
/// assert!(g10 > g100, "Gamma should decrease with more trials");
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct SqrtGamma {
|
||||
gamma_factor: f64,
|
||||
gamma_max: f64,
|
||||
}
|
||||
|
||||
impl SqrtGamma {
|
||||
/// Creates a new square root gamma strategy.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `gamma_factor` - The factor controlling how quickly gamma decreases.
|
||||
/// Higher values mean more "good" trials at any given point.
|
||||
/// * `gamma_max` - The maximum gamma value (used when `n_trials` is small).
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns `Error::InvalidGamma` if:
|
||||
/// - `gamma_factor` is not positive
|
||||
/// - `gamma_max` is not in (0.0, 1.0)
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::sampler::tpe::SqrtGamma;
|
||||
///
|
||||
/// let strategy = SqrtGamma::new(1.0, 0.25).unwrap();
|
||||
/// ```
|
||||
pub fn new(gamma_factor: f64, gamma_max: f64) -> crate::Result<Self> {
|
||||
if gamma_factor <= 0.0 {
|
||||
return Err(Error::InvalidGamma(gamma_factor));
|
||||
}
|
||||
if gamma_max <= 0.0 || gamma_max >= 1.0 {
|
||||
return Err(Error::InvalidGamma(gamma_max));
|
||||
}
|
||||
Ok(Self {
|
||||
gamma_factor,
|
||||
gamma_max,
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the gamma factor.
|
||||
#[must_use]
|
||||
pub fn gamma_factor(&self) -> f64 {
|
||||
self.gamma_factor
|
||||
}
|
||||
|
||||
/// Returns the maximum gamma value.
|
||||
#[must_use]
|
||||
pub fn gamma_max(&self) -> f64 {
|
||||
self.gamma_max
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SqrtGamma {
|
||||
/// Creates a square root gamma strategy with default values:
|
||||
/// - `gamma_factor`: 1.0
|
||||
/// - `gamma_max`: 0.25
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
gamma_factor: 1.0,
|
||||
gamma_max: 0.25,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl GammaStrategy for SqrtGamma {
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
fn gamma(&self, n_trials: usize) -> f64 {
|
||||
if n_trials == 0 {
|
||||
return self.gamma_max;
|
||||
}
|
||||
let n_good = (self.gamma_factor / (n_trials as f64).sqrt()).max(1.0);
|
||||
(n_good / n_trials as f64).min(self.gamma_max)
|
||||
}
|
||||
|
||||
fn clone_box(&self) -> Box<dyn GammaStrategy> {
|
||||
Box::new(*self)
|
||||
}
|
||||
}
|
||||
|
||||
/// A Hyperopt-style gamma strategy.
|
||||
///
|
||||
/// This strategy computes gamma as `min(gamma_max, (gamma_base + 1) / n_trials)`,
|
||||
/// which is inspired by the original Hyperopt TPE implementation.
|
||||
///
|
||||
/// # Formula
|
||||
///
|
||||
/// ```text
|
||||
/// gamma = min(gamma_max, (gamma_base + 1) / n_trials)
|
||||
/// ```
|
||||
///
|
||||
/// When `n_trials` is 0, returns `gamma_max`.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::sampler::tpe::{GammaStrategy, HyperoptGamma};
|
||||
///
|
||||
/// // With gamma_base=24 and gamma_max=0.5:
|
||||
/// // - At n=25: gamma = min(0.5, 25/25) = 0.5 (capped)
|
||||
/// // - At n=100: gamma = min(0.5, 25/100) = 0.25
|
||||
/// let strategy = HyperoptGamma::new(24.0, 0.5).unwrap();
|
||||
///
|
||||
/// // Early trials have higher gamma
|
||||
/// let g50 = strategy.gamma(50);
|
||||
/// let g200 = strategy.gamma(200);
|
||||
/// assert!(g50 > g200, "Gamma should decrease with more trials");
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct HyperoptGamma {
|
||||
gamma_base: f64,
|
||||
gamma_max: f64,
|
||||
}
|
||||
|
||||
impl HyperoptGamma {
|
||||
/// Creates a new Hyperopt-style gamma strategy.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `gamma_base` - The base value added to 1 in the numerator.
|
||||
/// * `gamma_max` - The maximum gamma value.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns `Error::InvalidGamma` if:
|
||||
/// - `gamma_base` is negative
|
||||
/// - `gamma_max` is not in (0.0, 1.0)
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::sampler::tpe::HyperoptGamma;
|
||||
///
|
||||
/// let strategy = HyperoptGamma::new(24.0, 0.25).unwrap();
|
||||
/// ```
|
||||
pub fn new(gamma_base: f64, gamma_max: f64) -> crate::Result<Self> {
|
||||
if gamma_base < 0.0 {
|
||||
return Err(Error::InvalidGamma(gamma_base));
|
||||
}
|
||||
if gamma_max <= 0.0 || gamma_max >= 1.0 {
|
||||
return Err(Error::InvalidGamma(gamma_max));
|
||||
}
|
||||
Ok(Self {
|
||||
gamma_base,
|
||||
gamma_max,
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the gamma base value.
|
||||
#[must_use]
|
||||
pub fn gamma_base(&self) -> f64 {
|
||||
self.gamma_base
|
||||
}
|
||||
|
||||
/// Returns the maximum gamma value.
|
||||
#[must_use]
|
||||
pub fn gamma_max(&self) -> f64 {
|
||||
self.gamma_max
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for HyperoptGamma {
|
||||
/// Creates a Hyperopt-style gamma strategy with default values:
|
||||
/// - `gamma_base`: 24.0
|
||||
/// - `gamma_max`: 0.25
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
gamma_base: 24.0,
|
||||
gamma_max: 0.25,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl GammaStrategy for HyperoptGamma {
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
fn gamma(&self, n_trials: usize) -> f64 {
|
||||
if n_trials == 0 {
|
||||
return self.gamma_max;
|
||||
}
|
||||
((self.gamma_base + 1.0) / n_trials as f64).min(self.gamma_max)
|
||||
}
|
||||
|
||||
fn clone_box(&self) -> Box<dyn GammaStrategy> {
|
||||
Box::new(*self)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::sampler::tpe::TpeSampler;
|
||||
|
||||
#[test]
|
||||
fn test_fixed_gamma_default() {
|
||||
let strategy = FixedGamma::default();
|
||||
assert!((strategy.gamma(0) - 0.25).abs() < f64::EPSILON);
|
||||
assert!((strategy.gamma(100) - 0.25).abs() < f64::EPSILON);
|
||||
assert!((strategy.value() - 0.25).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fixed_gamma_custom() {
|
||||
let strategy = FixedGamma::new(0.15).unwrap();
|
||||
assert!((strategy.gamma(0) - 0.15).abs() < f64::EPSILON);
|
||||
assert!((strategy.gamma(50) - 0.15).abs() < f64::EPSILON);
|
||||
assert!((strategy.gamma(1000) - 0.15).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_fixed_gamma_invalid() {
|
||||
assert!(FixedGamma::new(0.0).is_err());
|
||||
assert!(FixedGamma::new(1.0).is_err());
|
||||
assert!(FixedGamma::new(-0.1).is_err());
|
||||
assert!(FixedGamma::new(1.5).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_linear_gamma_default() {
|
||||
let strategy = LinearGamma::default();
|
||||
assert!((strategy.gamma(0) - 0.10).abs() < f64::EPSILON);
|
||||
assert!((strategy.gamma(50) - 0.175).abs() < f64::EPSILON); // midpoint
|
||||
assert!((strategy.gamma(100) - 0.25).abs() < f64::EPSILON);
|
||||
assert!((strategy.gamma(200) - 0.25).abs() < f64::EPSILON); // capped
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_linear_gamma_custom() {
|
||||
let strategy = LinearGamma::new(0.1, 0.4, 100).unwrap();
|
||||
assert!((strategy.gamma(0) - 0.1).abs() < f64::EPSILON);
|
||||
assert!((strategy.gamma(50) - 0.25).abs() < f64::EPSILON);
|
||||
assert!((strategy.gamma(100) - 0.4).abs() < f64::EPSILON);
|
||||
assert!((strategy.gamma(200) - 0.4).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_linear_gamma_invalid() {
|
||||
assert!(LinearGamma::new(0.0, 0.5, 100).is_err());
|
||||
assert!(LinearGamma::new(0.1, 1.0, 100).is_err());
|
||||
assert!(LinearGamma::new(0.5, 0.2, 100).is_err()); // min > max
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sqrt_gamma_default() {
|
||||
let strategy = SqrtGamma::default();
|
||||
// At n=0, returns gamma_max
|
||||
assert!((strategy.gamma(0) - 0.25).abs() < f64::EPSILON);
|
||||
|
||||
// gamma decreases with more trials
|
||||
let g10 = strategy.gamma(10);
|
||||
let g100 = strategy.gamma(100);
|
||||
assert!(g10 > g100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sqrt_gamma_custom() {
|
||||
let strategy = SqrtGamma::new(2.0, 0.5).unwrap();
|
||||
assert!((strategy.gamma(0) - 0.5).abs() < f64::EPSILON);
|
||||
|
||||
// At n=4: n_good = max(1, 2/2) = 1, gamma = 1/4 = 0.25
|
||||
let g4 = strategy.gamma(4);
|
||||
assert!((g4 - 0.25).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sqrt_gamma_invalid() {
|
||||
assert!(SqrtGamma::new(0.0, 0.25).is_err()); // factor must be positive
|
||||
assert!(SqrtGamma::new(-1.0, 0.25).is_err());
|
||||
assert!(SqrtGamma::new(1.0, 0.0).is_err());
|
||||
assert!(SqrtGamma::new(1.0, 1.0).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hyperopt_gamma_default() {
|
||||
let strategy = HyperoptGamma::default();
|
||||
// At n=0, returns gamma_max
|
||||
assert!((strategy.gamma(0) - 0.25).abs() < f64::EPSILON);
|
||||
|
||||
// At n=100: (24+1)/100 = 0.25, so capped to 0.25
|
||||
assert!((strategy.gamma(100) - 0.25).abs() < f64::EPSILON);
|
||||
|
||||
// At n=200: (24+1)/200 = 0.125
|
||||
assert!((strategy.gamma(200) - 0.125).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hyperopt_gamma_custom() {
|
||||
let strategy = HyperoptGamma::new(9.0, 0.5).unwrap();
|
||||
// At n=20: (9+1)/20 = 0.5, capped to 0.5
|
||||
assert!((strategy.gamma(20) - 0.5).abs() < f64::EPSILON);
|
||||
|
||||
// At n=100: (9+1)/100 = 0.1
|
||||
assert!((strategy.gamma(100) - 0.1).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hyperopt_gamma_invalid() {
|
||||
assert!(HyperoptGamma::new(-1.0, 0.25).is_err());
|
||||
assert!(HyperoptGamma::new(24.0, 0.0).is_err());
|
||||
assert!(HyperoptGamma::new(24.0, 1.0).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gamma_strategy_clone_box() {
|
||||
let fixed: Box<dyn GammaStrategy> = Box::new(FixedGamma::new(0.3).unwrap());
|
||||
let cloned = fixed.clone();
|
||||
assert!((cloned.gamma(0) - 0.3).abs() < f64::EPSILON);
|
||||
|
||||
let linear: Box<dyn GammaStrategy> = Box::new(LinearGamma::default());
|
||||
let cloned = linear.clone();
|
||||
assert!((cloned.gamma(0) - 0.10).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tpe_with_linear_gamma_strategy() {
|
||||
let sampler = TpeSampler::builder()
|
||||
.gamma_strategy(LinearGamma::new(0.1, 0.3, 50).unwrap())
|
||||
.n_startup_trials(5)
|
||||
.seed(42)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Verify the strategy is applied
|
||||
let g = sampler.gamma_strategy().gamma(25);
|
||||
assert!((g - 0.2).abs() < f64::EPSILON); // midpoint of 0.1 to 0.3
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gamma_overrides_gamma_strategy() {
|
||||
// When gamma() is called after gamma_strategy(), it should take precedence
|
||||
let sampler = TpeSampler::builder()
|
||||
.gamma_strategy(SqrtGamma::default())
|
||||
.gamma(0.15) // This should override
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Should use fixed gamma of 0.15
|
||||
assert!((sampler.gamma_strategy().gamma(0) - 0.15).abs() < f64::EPSILON);
|
||||
assert!((sampler.gamma_strategy().gamma(100) - 0.15).abs() < f64::EPSILON);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_gamma_strategy_overrides_gamma() {
|
||||
// When gamma_strategy() is called after gamma(), it should take precedence
|
||||
let sampler = TpeSampler::builder()
|
||||
.gamma(0.15)
|
||||
.gamma_strategy(SqrtGamma::default()) // This should override
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Should use SqrtGamma - gamma decreases with trials
|
||||
let g10 = sampler.gamma_strategy().gamma(10);
|
||||
let g100 = sampler.gamma_strategy().gamma(100);
|
||||
assert!(g10 > g100, "SqrtGamma should decrease with more trials");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_custom_gamma_strategy() {
|
||||
#[derive(Debug, Clone)]
|
||||
struct DoubleGamma;
|
||||
|
||||
impl GammaStrategy for DoubleGamma {
|
||||
fn gamma(&self, n_trials: usize) -> f64 {
|
||||
// Double the trial count-based calculation, capped at 0.5
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
(0.01 * n_trials as f64).min(0.5)
|
||||
}
|
||||
|
||||
fn clone_box(&self) -> Box<dyn GammaStrategy> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
}
|
||||
|
||||
let sampler = TpeSampler::builder()
|
||||
.gamma_strategy(DoubleGamma)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
assert!((sampler.gamma_strategy().gamma(10) - 0.1).abs() < f64::EPSILON);
|
||||
assert!((sampler.gamma_strategy().gamma(50) - 0.5).abs() < f64::EPSILON);
|
||||
assert!((sampler.gamma_strategy().gamma(100) - 0.5).abs() < f64::EPSILON);
|
||||
}
|
||||
}
|
||||
@@ -1,16 +0,0 @@
|
||||
//! Tree-Parzen Estimator (TPE) sampler implementation and utilities.
|
||||
//!
|
||||
//! This module provides TPE-based sampling for Bayesian optimization,
|
||||
//! including support for intersection search space calculation.
|
||||
|
||||
mod gamma;
|
||||
mod multivariate;
|
||||
mod sampler;
|
||||
pub mod search_space;
|
||||
|
||||
pub use gamma::{FixedGamma, GammaStrategy, HyperoptGamma, LinearGamma, SqrtGamma};
|
||||
pub use multivariate::{
|
||||
ConstantLiarStrategy, MultivariateTpeSampler, MultivariateTpeSamplerBuilder,
|
||||
};
|
||||
pub use sampler::{TpeSampler, TpeSamplerBuilder};
|
||||
pub use search_space::{GroupDecomposedSearchSpace, IntersectionSearchSpace};
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+312
-1664
File diff suppressed because it is too large
Load Diff
+613
-232
@@ -5,58 +5,14 @@ use std::sync::Arc;
|
||||
|
||||
use parking_lot::RwLock;
|
||||
|
||||
use crate::distribution::Distribution;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::distribution::{
|
||||
CategoricalDistribution, Distribution, FloatDistribution, IntDistribution,
|
||||
};
|
||||
use crate::error::{Result, TpeError};
|
||||
use crate::param::ParamValue;
|
||||
use crate::parameter::{ParamId, Parameter};
|
||||
use crate::pruner::Pruner;
|
||||
use crate::sampler::{CompletedTrial, Sampler};
|
||||
use crate::types::TrialState;
|
||||
|
||||
/// A user attribute value that can be stored on a trial.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
pub enum AttrValue {
|
||||
/// A floating-point attribute.
|
||||
Float(f64),
|
||||
/// An integer attribute.
|
||||
Int(i64),
|
||||
/// A string attribute.
|
||||
String(String),
|
||||
/// A boolean attribute.
|
||||
Bool(bool),
|
||||
}
|
||||
|
||||
impl From<f64> for AttrValue {
|
||||
fn from(v: f64) -> Self {
|
||||
Self::Float(v)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<i64> for AttrValue {
|
||||
fn from(v: i64) -> Self {
|
||||
Self::Int(v)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<String> for AttrValue {
|
||||
fn from(v: String) -> Self {
|
||||
Self::String(v)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for AttrValue {
|
||||
fn from(v: &str) -> Self {
|
||||
Self::String(v.to_owned())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<bool> for AttrValue {
|
||||
fn from(v: bool) -> Self {
|
||||
Self::Bool(v)
|
||||
}
|
||||
}
|
||||
|
||||
/// A trial represents a single evaluation of the objective function.
|
||||
///
|
||||
/// Each trial has a unique ID and stores the sampled parameters along with
|
||||
@@ -71,43 +27,25 @@ pub struct Trial {
|
||||
id: u64,
|
||||
/// Current state of the trial.
|
||||
state: TrialState,
|
||||
/// Sampled parameter values, keyed by parameter id.
|
||||
params: HashMap<ParamId, ParamValue>,
|
||||
/// Parameter distributions, keyed by parameter id.
|
||||
distributions: HashMap<ParamId, Distribution>,
|
||||
/// Human-readable labels for parameters, keyed by parameter id.
|
||||
param_labels: HashMap<ParamId, String>,
|
||||
/// Sampled parameter values, keyed by parameter name.
|
||||
params: HashMap<String, ParamValue>,
|
||||
/// Parameter distributions, keyed by parameter name.
|
||||
distributions: HashMap<String, Distribution>,
|
||||
/// The sampler to use for generating parameter values.
|
||||
sampler: Option<Arc<dyn Sampler>>,
|
||||
/// Access to the history of completed trials (shared with Study).
|
||||
history: Option<Arc<RwLock<Vec<CompletedTrial<f64>>>>>,
|
||||
/// Intermediate objective values reported at each step.
|
||||
intermediate_values: Vec<(u64, f64)>,
|
||||
/// The pruner used to decide whether to stop this trial early.
|
||||
pruner: Option<Arc<dyn Pruner>>,
|
||||
/// User-defined attributes for logging, debugging, and analysis.
|
||||
user_attrs: HashMap<String, AttrValue>,
|
||||
/// Pre-filled parameter values from enqueue (used instead of sampling).
|
||||
fixed_params: HashMap<ParamId, ParamValue>,
|
||||
/// Constraint values for this trial (<=0.0 means feasible).
|
||||
constraint_values: Vec<f64>,
|
||||
}
|
||||
|
||||
impl core::fmt::Debug for Trial {
|
||||
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
|
||||
impl std::fmt::Debug for Trial {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("Trial")
|
||||
.field("id", &self.id)
|
||||
.field("state", &self.state)
|
||||
.field("params", &self.params)
|
||||
.field("distributions", &self.distributions)
|
||||
.field("param_labels", &self.param_labels)
|
||||
.field("has_sampler", &self.sampler.is_some())
|
||||
.field("has_history", &self.history.is_some())
|
||||
.field("intermediate_values", &self.intermediate_values)
|
||||
.field("has_pruner", &self.pruner.is_some())
|
||||
.field("user_attrs", &self.user_attrs)
|
||||
.field("fixed_params", &self.fixed_params)
|
||||
.field("constraint_values", &self.constraint_values)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -117,7 +55,7 @@ impl Trial {
|
||||
///
|
||||
/// The trial starts in the `Running` state with no parameters sampled.
|
||||
/// This constructor creates a trial without a sampler, which will use
|
||||
/// local random sampling for suggest methods.
|
||||
/// local random sampling for suggest_* methods.
|
||||
///
|
||||
/// For trials that use the study's sampler, use `Trial::with_sampler` instead.
|
||||
///
|
||||
@@ -133,21 +71,14 @@ impl Trial {
|
||||
/// let trial = Trial::new(0);
|
||||
/// assert_eq!(trial.id(), 0);
|
||||
/// ```
|
||||
#[must_use]
|
||||
pub fn new(id: u64) -> Self {
|
||||
Self {
|
||||
id,
|
||||
state: TrialState::Running,
|
||||
params: HashMap::new(),
|
||||
distributions: HashMap::new(),
|
||||
param_labels: HashMap::new(),
|
||||
sampler: None,
|
||||
history: None,
|
||||
intermediate_values: Vec::new(),
|
||||
pruner: None,
|
||||
user_attrs: HashMap::new(),
|
||||
fixed_params: HashMap::new(),
|
||||
constraint_values: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -165,36 +96,21 @@ impl Trial {
|
||||
id: u64,
|
||||
sampler: Arc<dyn Sampler>,
|
||||
history: Arc<RwLock<Vec<CompletedTrial<f64>>>>,
|
||||
pruner: Arc<dyn Pruner>,
|
||||
) -> Self {
|
||||
Self {
|
||||
id,
|
||||
state: TrialState::Running,
|
||||
params: HashMap::new(),
|
||||
distributions: HashMap::new(),
|
||||
param_labels: HashMap::new(),
|
||||
sampler: Some(sampler),
|
||||
history: Some(history),
|
||||
intermediate_values: Vec::new(),
|
||||
pruner: Some(pruner),
|
||||
user_attrs: HashMap::new(),
|
||||
fixed_params: HashMap::new(),
|
||||
constraint_values: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets pre-filled parameters on this trial.
|
||||
///
|
||||
/// When `suggest_param` is called for a parameter that has a fixed value,
|
||||
/// the fixed value is used instead of sampling.
|
||||
pub(crate) fn set_fixed_params(&mut self, params: HashMap<ParamId, ParamValue>) {
|
||||
self.fixed_params = params;
|
||||
}
|
||||
|
||||
/// Samples a value from the given distribution using the sampler.
|
||||
///
|
||||
/// If the trial has a sampler, it delegates to the sampler's sample method
|
||||
/// with the history of completed trials. Otherwise, it uses the `RandomSampler`
|
||||
/// with the history of completed trials. Otherwise, it uses the RandomSampler
|
||||
/// as a fallback.
|
||||
fn sample_value(&self, distribution: &Distribution) -> ParamValue {
|
||||
if let (Some(sampler), Some(history)) = (&self.sampler, &self.history) {
|
||||
@@ -202,115 +118,32 @@ impl Trial {
|
||||
sampler.sample(distribution, self.id, &history_guard)
|
||||
} else {
|
||||
// Fallback to RandomSampler when no sampler is configured
|
||||
use crate::sampler::random::RandomSampler;
|
||||
use crate::sampler::RandomSampler;
|
||||
let fallback = RandomSampler::new();
|
||||
fallback.sample(distribution, self.id, &[])
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the unique ID of this trial.
|
||||
#[must_use]
|
||||
pub fn id(&self) -> u64 {
|
||||
self.id
|
||||
}
|
||||
|
||||
/// Returns the current state of this trial.
|
||||
#[must_use]
|
||||
pub fn state(&self) -> TrialState {
|
||||
self.state
|
||||
}
|
||||
|
||||
/// Returns a reference to the sampled parameters.
|
||||
#[must_use]
|
||||
pub fn params(&self) -> &HashMap<ParamId, ParamValue> {
|
||||
pub fn params(&self) -> &HashMap<String, ParamValue> {
|
||||
&self.params
|
||||
}
|
||||
|
||||
/// Returns a reference to the parameter distributions.
|
||||
#[must_use]
|
||||
pub fn distributions(&self) -> &HashMap<ParamId, Distribution> {
|
||||
pub fn distributions(&self) -> &HashMap<String, Distribution> {
|
||||
&self.distributions
|
||||
}
|
||||
|
||||
/// Returns a reference to the parameter labels.
|
||||
#[must_use]
|
||||
pub fn param_labels(&self) -> &HashMap<ParamId, String> {
|
||||
&self.param_labels
|
||||
}
|
||||
|
||||
/// Reports an intermediate objective value at a given step.
|
||||
///
|
||||
/// Steps should be monotonically increasing (e.g., epoch number).
|
||||
/// Duplicate steps overwrite the previous value.
|
||||
pub fn report(&mut self, step: u64, value: f64) {
|
||||
if let Some(entry) = self
|
||||
.intermediate_values
|
||||
.iter_mut()
|
||||
.find(|(s, _)| *s == step)
|
||||
{
|
||||
entry.1 = value;
|
||||
} else {
|
||||
self.intermediate_values.push((step, value));
|
||||
}
|
||||
}
|
||||
|
||||
/// Ask whether this trial should be pruned at the current step.
|
||||
///
|
||||
/// Returns `true` if the pruner recommends stopping this trial.
|
||||
/// The caller should return `Err(TrialPruned)` from the objective.
|
||||
#[must_use]
|
||||
pub fn should_prune(&self) -> bool {
|
||||
let (Some(pruner), Some(history)) = (&self.pruner, &self.history) else {
|
||||
return false;
|
||||
};
|
||||
let Some(&(step, _)) = self.intermediate_values.last() else {
|
||||
return false;
|
||||
};
|
||||
let history_guard = history.read();
|
||||
let prune = pruner.should_prune(self.id, step, &self.intermediate_values, &history_guard);
|
||||
if prune {
|
||||
trace_info!(trial_id = self.id, step, "pruner recommends stopping");
|
||||
}
|
||||
prune
|
||||
}
|
||||
|
||||
/// Returns all intermediate values reported so far.
|
||||
#[must_use]
|
||||
pub fn intermediate_values(&self) -> &[(u64, f64)] {
|
||||
&self.intermediate_values
|
||||
}
|
||||
|
||||
/// Sets a user attribute on this trial.
|
||||
pub fn set_user_attr(&mut self, key: impl Into<String>, value: impl Into<AttrValue>) {
|
||||
self.user_attrs.insert(key.into(), value.into());
|
||||
}
|
||||
|
||||
/// Gets a user attribute by key.
|
||||
#[must_use]
|
||||
pub fn user_attr(&self, key: &str) -> Option<&AttrValue> {
|
||||
self.user_attrs.get(key)
|
||||
}
|
||||
|
||||
/// Returns all user attributes.
|
||||
#[must_use]
|
||||
pub fn user_attrs(&self) -> &HashMap<String, AttrValue> {
|
||||
&self.user_attrs
|
||||
}
|
||||
|
||||
/// Sets constraint values for this trial.
|
||||
///
|
||||
/// Each value represents a constraint; a value <= 0.0 means the constraint
|
||||
/// is satisfied (feasible). A value > 0.0 means the constraint is violated.
|
||||
pub fn set_constraints(&mut self, values: Vec<f64>) {
|
||||
self.constraint_values = values;
|
||||
}
|
||||
|
||||
/// Returns the constraint values for this trial.
|
||||
#[must_use]
|
||||
pub fn constraint_values(&self) -> &[f64] {
|
||||
&self.constraint_values
|
||||
}
|
||||
|
||||
/// Sets the trial state to Complete.
|
||||
pub(crate) fn set_complete(&mut self) {
|
||||
self.state = TrialState::Complete;
|
||||
@@ -321,87 +154,635 @@ impl Trial {
|
||||
self.state = TrialState::Failed;
|
||||
}
|
||||
|
||||
/// Sets the trial state to Pruned.
|
||||
pub(crate) fn set_pruned(&mut self) {
|
||||
self.state = TrialState::Pruned;
|
||||
}
|
||||
|
||||
/// Suggests a parameter value using a [`Parameter`] definition.
|
||||
/// Suggests a float parameter with the given bounds.
|
||||
///
|
||||
/// This is the primary entry point for sampling parameters. It handles
|
||||
/// validation, caching, conflict detection, sampling, and conversion.
|
||||
/// If the parameter has already been sampled with the same bounds, the cached value is returned.
|
||||
/// If the parameter was sampled with different bounds, a `ParameterConflict` error is returned.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `param` - The parameter definition.
|
||||
/// * `name` - The name of the parameter.
|
||||
/// * `low` - The lower bound (inclusive).
|
||||
/// * `high` - The upper bound (inclusive).
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an error if:
|
||||
/// - The parameter fails validation
|
||||
/// - The parameter conflicts with a previously suggested parameter of the same id
|
||||
/// - Sampling or conversion fails
|
||||
/// Returns `InvalidBounds` if `low > high`.
|
||||
/// Returns `ParameterConflict` if the parameter was previously sampled with different bounds.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::Trial;
|
||||
/// use optimizer::parameter::{BoolParam, FloatParam, IntParam, Parameter};
|
||||
///
|
||||
/// let x_param = FloatParam::new(0.0, 1.0);
|
||||
/// let n_param = IntParam::new(1, 10);
|
||||
/// let flag_param = BoolParam::new();
|
||||
///
|
||||
/// let mut trial = Trial::new(0);
|
||||
/// let x = trial.suggest_float("x", 0.0, 1.0).unwrap();
|
||||
/// assert!(x >= 0.0 && x <= 1.0);
|
||||
///
|
||||
/// let x = trial.suggest_param(&x_param).unwrap();
|
||||
/// let n = trial.suggest_param(&n_param).unwrap();
|
||||
/// let flag = trial.suggest_param(&flag_param).unwrap();
|
||||
/// // Calling again with same bounds returns cached value
|
||||
/// let x2 = trial.suggest_float("x", 0.0, 1.0).unwrap();
|
||||
/// assert_eq!(x, x2);
|
||||
/// ```
|
||||
pub fn suggest_param<P: Parameter>(&mut self, param: &P) -> Result<P::Value> {
|
||||
param.validate()?;
|
||||
pub fn suggest_float(&mut self, name: impl Into<String>, low: f64, high: f64) -> Result<f64> {
|
||||
if low > high {
|
||||
return Err(TpeError::InvalidBounds { low, high });
|
||||
}
|
||||
|
||||
let param_id = param.id();
|
||||
let distribution = param.distribution();
|
||||
let name = name.into();
|
||||
let distribution = FloatDistribution {
|
||||
low,
|
||||
high,
|
||||
log_scale: false,
|
||||
step: None,
|
||||
};
|
||||
|
||||
// Check if parameter already exists
|
||||
if let Some(existing_dist) = self.distributions.get(¶m_id) {
|
||||
if *existing_dist == distribution {
|
||||
if let Some(existing_dist) = self.distributions.get(&name) {
|
||||
// Verify the distribution matches
|
||||
if let Distribution::Float(existing) = existing_dist
|
||||
&& existing.low == low
|
||||
&& existing.high == high
|
||||
&& !existing.log_scale
|
||||
&& existing.step.is_none()
|
||||
{
|
||||
// Same distribution, return cached value
|
||||
if let Some(value) = self.params.get(¶m_id) {
|
||||
return param.cast_param_value(value);
|
||||
if let Some(ParamValue::Float(value)) = self.params.get(&name) {
|
||||
return Ok(*value);
|
||||
}
|
||||
}
|
||||
// Distribution exists but doesn't match
|
||||
return Err(Error::ParameterConflict {
|
||||
name: param.label(),
|
||||
reason: "parameter was previously sampled with different configuration or type"
|
||||
return Err(TpeError::ParameterConflict {
|
||||
name,
|
||||
reason: "parameter was previously sampled with different bounds or type"
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// Check for a pre-filled (enqueued) value for this parameter
|
||||
let value = if let Some(fixed_value) = self.fixed_params.remove(¶m_id) {
|
||||
fixed_value
|
||||
} else {
|
||||
// Sample using the sampler
|
||||
self.sample_value(&distribution)
|
||||
// Sample using the sampler
|
||||
let dist = Distribution::Float(distribution);
|
||||
let value = match self.sample_value(&dist) {
|
||||
ParamValue::Float(v) => v,
|
||||
_ => unreachable!("Float distribution should return Float value"),
|
||||
};
|
||||
|
||||
let result = param.cast_param_value(&value)?;
|
||||
// Store distribution and value
|
||||
self.distributions.insert(name.clone(), dist);
|
||||
self.params.insert(name, ParamValue::Float(value));
|
||||
|
||||
trace_debug!(
|
||||
trial_id = self.id,
|
||||
param = %param.label(),
|
||||
value = %value,
|
||||
"parameter sampled"
|
||||
);
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
// Store distribution, value, and label
|
||||
self.distributions.insert(param_id, distribution);
|
||||
self.params.insert(param_id, value);
|
||||
self.param_labels.insert(param_id, param.label());
|
||||
/// Suggests a float parameter sampled on a logarithmic scale.
|
||||
///
|
||||
/// The value is sampled uniformly in log space, which is useful for parameters
|
||||
/// that span multiple orders of magnitude (e.g., learning rates).
|
||||
///
|
||||
/// If the parameter has already been sampled with the same bounds and log_scale=true,
|
||||
/// the cached value is returned. If the parameter was sampled with different configuration,
|
||||
/// a `ParameterConflict` error is returned.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `name` - The name of the parameter.
|
||||
/// * `low` - The lower bound (inclusive, must be positive).
|
||||
/// * `high` - The upper bound (inclusive).
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns `InvalidLogBounds` if `low <= 0`.
|
||||
/// Returns `InvalidBounds` if `low > high`.
|
||||
/// Returns `ParameterConflict` if the parameter was previously sampled with different configuration.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::Trial;
|
||||
///
|
||||
/// let mut trial = Trial::new(0);
|
||||
/// let lr = trial
|
||||
/// .suggest_float_log("learning_rate", 1e-5, 1e-1)
|
||||
/// .unwrap();
|
||||
/// assert!(lr >= 1e-5 && lr <= 1e-1);
|
||||
///
|
||||
/// // Calling again with same bounds returns cached value
|
||||
/// let lr2 = trial
|
||||
/// .suggest_float_log("learning_rate", 1e-5, 1e-1)
|
||||
/// .unwrap();
|
||||
/// assert_eq!(lr, lr2);
|
||||
/// ```
|
||||
pub fn suggest_float_log(
|
||||
&mut self,
|
||||
name: impl Into<String>,
|
||||
low: f64,
|
||||
high: f64,
|
||||
) -> Result<f64> {
|
||||
if low <= 0.0 {
|
||||
return Err(TpeError::InvalidLogBounds);
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
if low > high {
|
||||
return Err(TpeError::InvalidBounds { low, high });
|
||||
}
|
||||
|
||||
let name = name.into();
|
||||
let distribution = FloatDistribution {
|
||||
low,
|
||||
high,
|
||||
log_scale: true,
|
||||
step: None,
|
||||
};
|
||||
|
||||
// Check if parameter already exists
|
||||
if let Some(existing_dist) = self.distributions.get(&name) {
|
||||
// Verify the distribution matches
|
||||
if let Distribution::Float(existing) = existing_dist
|
||||
&& existing.low == low
|
||||
&& existing.high == high
|
||||
&& existing.log_scale
|
||||
&& existing.step.is_none()
|
||||
{
|
||||
// Same distribution, return cached value
|
||||
if let Some(ParamValue::Float(value)) = self.params.get(&name) {
|
||||
return Ok(*value);
|
||||
}
|
||||
}
|
||||
// Distribution exists but doesn't match
|
||||
return Err(TpeError::ParameterConflict {
|
||||
name,
|
||||
reason: "parameter was previously sampled with different bounds or type"
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// Sample using the sampler (sampler handles log-scale transformation)
|
||||
let dist = Distribution::Float(distribution);
|
||||
let value = match self.sample_value(&dist) {
|
||||
ParamValue::Float(v) => v,
|
||||
_ => unreachable!("Float distribution should return Float value"),
|
||||
};
|
||||
|
||||
// Store distribution and value
|
||||
self.distributions.insert(name.clone(), dist);
|
||||
self.params.insert(name, ParamValue::Float(value));
|
||||
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
/// Suggests a float parameter that snaps to a step grid.
|
||||
///
|
||||
/// The value is sampled from the discrete set {low, low + step, low + 2*step, ...}
|
||||
/// where each value is <= high.
|
||||
///
|
||||
/// If the parameter has already been sampled with the same configuration,
|
||||
/// the cached value is returned. If the parameter was sampled with different configuration,
|
||||
/// a `ParameterConflict` error is returned.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `name` - The name of the parameter.
|
||||
/// * `low` - The lower bound (inclusive).
|
||||
/// * `high` - The upper bound (inclusive).
|
||||
/// * `step` - The step size (must be positive).
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns `InvalidStep` if `step <= 0`.
|
||||
/// Returns `InvalidBounds` if `low > high`.
|
||||
/// Returns `ParameterConflict` if the parameter was previously sampled with different configuration.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::Trial;
|
||||
///
|
||||
/// let mut trial = Trial::new(0);
|
||||
/// let x = trial.suggest_float_step("x", 0.0, 1.0, 0.25).unwrap();
|
||||
/// // x will be one of: 0.0, 0.25, 0.5, 0.75, 1.0
|
||||
/// assert!(x >= 0.0 && x <= 1.0);
|
||||
/// assert!((x / 0.25).fract().abs() < 1e-10 || (x / 0.25).fract().abs() > 1.0 - 1e-10);
|
||||
///
|
||||
/// // Calling again with same bounds returns cached value
|
||||
/// let x2 = trial.suggest_float_step("x", 0.0, 1.0, 0.25).unwrap();
|
||||
/// assert_eq!(x, x2);
|
||||
/// ```
|
||||
pub fn suggest_float_step(
|
||||
&mut self,
|
||||
name: impl Into<String>,
|
||||
low: f64,
|
||||
high: f64,
|
||||
step: f64,
|
||||
) -> Result<f64> {
|
||||
if step <= 0.0 {
|
||||
return Err(TpeError::InvalidStep);
|
||||
}
|
||||
|
||||
if low > high {
|
||||
return Err(TpeError::InvalidBounds { low, high });
|
||||
}
|
||||
|
||||
let name = name.into();
|
||||
let distribution = FloatDistribution {
|
||||
low,
|
||||
high,
|
||||
log_scale: false,
|
||||
step: Some(step),
|
||||
};
|
||||
|
||||
// Check if parameter already exists
|
||||
if let Some(existing_dist) = self.distributions.get(&name) {
|
||||
// Verify the distribution matches
|
||||
if let Distribution::Float(existing) = existing_dist
|
||||
&& existing.low == low
|
||||
&& existing.high == high
|
||||
&& !existing.log_scale
|
||||
&& existing.step == Some(step)
|
||||
{
|
||||
// Same distribution, return cached value
|
||||
if let Some(ParamValue::Float(value)) = self.params.get(&name) {
|
||||
return Ok(*value);
|
||||
}
|
||||
}
|
||||
// Distribution exists but doesn't match
|
||||
return Err(TpeError::ParameterConflict {
|
||||
name,
|
||||
reason: "parameter was previously sampled with different bounds or type"
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// Sample using the sampler (sampler handles step-grid)
|
||||
let dist = Distribution::Float(distribution);
|
||||
let value = match self.sample_value(&dist) {
|
||||
ParamValue::Float(v) => v,
|
||||
_ => unreachable!("Float distribution should return Float value"),
|
||||
};
|
||||
|
||||
// Store distribution and value
|
||||
self.distributions.insert(name.clone(), dist);
|
||||
self.params.insert(name, ParamValue::Float(value));
|
||||
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
/// Suggests an integer parameter with the given bounds.
|
||||
///
|
||||
/// The value is sampled uniformly from the range [low, high] inclusive.
|
||||
///
|
||||
/// If the parameter has already been sampled with the same bounds, the cached value is returned.
|
||||
/// If the parameter was sampled with different bounds, a `ParameterConflict` error is returned.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `name` - The name of the parameter.
|
||||
/// * `low` - The lower bound (inclusive).
|
||||
/// * `high` - The upper bound (inclusive).
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns `InvalidBounds` if `low > high`.
|
||||
/// Returns `ParameterConflict` if the parameter was previously sampled with different bounds.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::Trial;
|
||||
///
|
||||
/// let mut trial = Trial::new(0);
|
||||
/// let n = trial.suggest_int("n_layers", 1, 10).unwrap();
|
||||
/// assert!(n >= 1 && n <= 10);
|
||||
///
|
||||
/// // Calling again with same bounds returns cached value
|
||||
/// let n2 = trial.suggest_int("n_layers", 1, 10).unwrap();
|
||||
/// assert_eq!(n, n2);
|
||||
/// ```
|
||||
pub fn suggest_int(&mut self, name: impl Into<String>, low: i64, high: i64) -> Result<i64> {
|
||||
if low > high {
|
||||
return Err(TpeError::InvalidBounds {
|
||||
low: low as f64,
|
||||
high: high as f64,
|
||||
});
|
||||
}
|
||||
|
||||
let name = name.into();
|
||||
let distribution = IntDistribution {
|
||||
low,
|
||||
high,
|
||||
log_scale: false,
|
||||
step: None,
|
||||
};
|
||||
|
||||
// Check if parameter already exists
|
||||
if let Some(existing_dist) = self.distributions.get(&name) {
|
||||
// Verify the distribution matches
|
||||
if let Distribution::Int(existing) = existing_dist
|
||||
&& existing.low == low
|
||||
&& existing.high == high
|
||||
&& !existing.log_scale
|
||||
&& existing.step.is_none()
|
||||
{
|
||||
// Same distribution, return cached value
|
||||
if let Some(ParamValue::Int(value)) = self.params.get(&name) {
|
||||
return Ok(*value);
|
||||
}
|
||||
}
|
||||
// Distribution exists but doesn't match
|
||||
return Err(TpeError::ParameterConflict {
|
||||
name,
|
||||
reason: "parameter was previously sampled with different bounds or type"
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// Sample using the sampler
|
||||
let dist = Distribution::Int(distribution);
|
||||
let value = match self.sample_value(&dist) {
|
||||
ParamValue::Int(v) => v,
|
||||
_ => unreachable!("Int distribution should return Int value"),
|
||||
};
|
||||
|
||||
// Store distribution and value
|
||||
self.distributions.insert(name.clone(), dist);
|
||||
self.params.insert(name, ParamValue::Int(value));
|
||||
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
/// Suggests an integer parameter sampled on a logarithmic scale.
|
||||
///
|
||||
/// The value is sampled uniformly in log space, which is useful for parameters
|
||||
/// that span multiple orders of magnitude (e.g., batch sizes).
|
||||
///
|
||||
/// If the parameter has already been sampled with the same bounds and log_scale=true,
|
||||
/// the cached value is returned. If the parameter was sampled with different configuration,
|
||||
/// a `ParameterConflict` error is returned.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `name` - The name of the parameter.
|
||||
/// * `low` - The lower bound (inclusive, must be >= 1).
|
||||
/// * `high` - The upper bound (inclusive).
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns `InvalidLogBounds` if `low < 1`.
|
||||
/// Returns `InvalidBounds` if `low > high`.
|
||||
/// Returns `ParameterConflict` if the parameter was previously sampled with different configuration.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::Trial;
|
||||
///
|
||||
/// let mut trial = Trial::new(0);
|
||||
/// let batch_size = trial.suggest_int_log("batch_size", 1, 1024).unwrap();
|
||||
/// assert!(batch_size >= 1 && batch_size <= 1024);
|
||||
///
|
||||
/// // Calling again with same bounds returns cached value
|
||||
/// let batch_size2 = trial.suggest_int_log("batch_size", 1, 1024).unwrap();
|
||||
/// assert_eq!(batch_size, batch_size2);
|
||||
/// ```
|
||||
pub fn suggest_int_log(&mut self, name: impl Into<String>, low: i64, high: i64) -> Result<i64> {
|
||||
if low < 1 {
|
||||
return Err(TpeError::InvalidLogBounds);
|
||||
}
|
||||
|
||||
if low > high {
|
||||
return Err(TpeError::InvalidBounds {
|
||||
low: low as f64,
|
||||
high: high as f64,
|
||||
});
|
||||
}
|
||||
|
||||
let name = name.into();
|
||||
let distribution = IntDistribution {
|
||||
low,
|
||||
high,
|
||||
log_scale: true,
|
||||
step: None,
|
||||
};
|
||||
|
||||
// Check if parameter already exists
|
||||
if let Some(existing_dist) = self.distributions.get(&name) {
|
||||
// Verify the distribution matches
|
||||
if let Distribution::Int(existing) = existing_dist
|
||||
&& existing.low == low
|
||||
&& existing.high == high
|
||||
&& existing.log_scale
|
||||
&& existing.step.is_none()
|
||||
{
|
||||
// Same distribution, return cached value
|
||||
if let Some(ParamValue::Int(value)) = self.params.get(&name) {
|
||||
return Ok(*value);
|
||||
}
|
||||
}
|
||||
// Distribution exists but doesn't match
|
||||
return Err(TpeError::ParameterConflict {
|
||||
name,
|
||||
reason: "parameter was previously sampled with different bounds or type"
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// Sample using the sampler (sampler handles log-scale transformation)
|
||||
let dist = Distribution::Int(distribution);
|
||||
let value = match self.sample_value(&dist) {
|
||||
ParamValue::Int(v) => v,
|
||||
_ => unreachable!("Int distribution should return Int value"),
|
||||
};
|
||||
|
||||
// Store distribution and value
|
||||
self.distributions.insert(name.clone(), dist);
|
||||
self.params.insert(name, ParamValue::Int(value));
|
||||
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
/// Suggests an integer parameter that snaps to a step grid.
|
||||
///
|
||||
/// The value is sampled from the discrete set {low, low + step, low + 2*step, ...}
|
||||
/// where each value is <= high.
|
||||
///
|
||||
/// If the parameter has already been sampled with the same configuration,
|
||||
/// the cached value is returned. If the parameter was sampled with different configuration,
|
||||
/// a `ParameterConflict` error is returned.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `name` - The name of the parameter.
|
||||
/// * `low` - The lower bound (inclusive).
|
||||
/// * `high` - The upper bound (inclusive).
|
||||
/// * `step` - The step size (must be positive).
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns `InvalidStep` if `step <= 0`.
|
||||
/// Returns `InvalidBounds` if `low > high`.
|
||||
/// Returns `ParameterConflict` if the parameter was previously sampled with different configuration.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::Trial;
|
||||
///
|
||||
/// let mut trial = Trial::new(0);
|
||||
/// let n = trial
|
||||
/// .suggest_int_step("n_estimators", 100, 500, 50)
|
||||
/// .unwrap();
|
||||
/// // n will be one of: 100, 150, 200, 250, 300, 350, 400, 450, 500
|
||||
/// assert!(n >= 100 && n <= 500);
|
||||
/// assert!((n - 100) % 50 == 0);
|
||||
///
|
||||
/// // Calling again with same bounds returns cached value
|
||||
/// let n2 = trial
|
||||
/// .suggest_int_step("n_estimators", 100, 500, 50)
|
||||
/// .unwrap();
|
||||
/// assert_eq!(n, n2);
|
||||
/// ```
|
||||
pub fn suggest_int_step(
|
||||
&mut self,
|
||||
name: impl Into<String>,
|
||||
low: i64,
|
||||
high: i64,
|
||||
step: i64,
|
||||
) -> Result<i64> {
|
||||
if step <= 0 {
|
||||
return Err(TpeError::InvalidStep);
|
||||
}
|
||||
|
||||
if low > high {
|
||||
return Err(TpeError::InvalidBounds {
|
||||
low: low as f64,
|
||||
high: high as f64,
|
||||
});
|
||||
}
|
||||
|
||||
let name = name.into();
|
||||
let distribution = IntDistribution {
|
||||
low,
|
||||
high,
|
||||
log_scale: false,
|
||||
step: Some(step),
|
||||
};
|
||||
|
||||
// Check if parameter already exists
|
||||
if let Some(existing_dist) = self.distributions.get(&name) {
|
||||
// Verify the distribution matches
|
||||
if let Distribution::Int(existing) = existing_dist
|
||||
&& existing.low == low
|
||||
&& existing.high == high
|
||||
&& !existing.log_scale
|
||||
&& existing.step == Some(step)
|
||||
{
|
||||
// Same distribution, return cached value
|
||||
if let Some(ParamValue::Int(value)) = self.params.get(&name) {
|
||||
return Ok(*value);
|
||||
}
|
||||
}
|
||||
// Distribution exists but doesn't match
|
||||
return Err(TpeError::ParameterConflict {
|
||||
name,
|
||||
reason: "parameter was previously sampled with different bounds or type"
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// Sample using the sampler (sampler handles step-grid)
|
||||
let dist = Distribution::Int(distribution);
|
||||
let value = match self.sample_value(&dist) {
|
||||
ParamValue::Int(v) => v,
|
||||
_ => unreachable!("Int distribution should return Int value"),
|
||||
};
|
||||
|
||||
// Store distribution and value
|
||||
self.distributions.insert(name.clone(), dist);
|
||||
self.params.insert(name, ParamValue::Int(value));
|
||||
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
/// Suggests a categorical parameter from the given choices.
|
||||
///
|
||||
/// The value is selected uniformly at random from the provided choices.
|
||||
/// Internally, the index of the selected choice is stored.
|
||||
///
|
||||
/// If the parameter has already been sampled with the same number of choices,
|
||||
/// the cached value (same index) is returned. If the parameter was sampled with
|
||||
/// a different number of choices, a `ParameterConflict` error is returned.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `name` - The name of the parameter.
|
||||
/// * `choices` - A slice of choices to select from.
|
||||
///
|
||||
/// # Type Parameters
|
||||
///
|
||||
/// * `T` - The type of the choices. Only requires `Clone`.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns `EmptyChoices` if `choices` is empty.
|
||||
/// Returns `ParameterConflict` if the parameter was previously sampled with a different number of choices.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ```
|
||||
/// use optimizer::Trial;
|
||||
///
|
||||
/// let mut trial = Trial::new(0);
|
||||
/// let optimizer = trial
|
||||
/// .suggest_categorical("optimizer", &["sgd", "adam", "rmsprop"])
|
||||
/// .unwrap();
|
||||
/// assert!(["sgd", "adam", "rmsprop"].contains(&optimizer));
|
||||
///
|
||||
/// // Calling again with same choices returns cached value
|
||||
/// let optimizer2 = trial
|
||||
/// .suggest_categorical("optimizer", &["sgd", "adam", "rmsprop"])
|
||||
/// .unwrap();
|
||||
/// assert_eq!(optimizer, optimizer2);
|
||||
/// ```
|
||||
pub fn suggest_categorical<T: Clone>(
|
||||
&mut self,
|
||||
name: impl Into<String>,
|
||||
choices: &[T],
|
||||
) -> Result<T> {
|
||||
if choices.is_empty() {
|
||||
return Err(TpeError::EmptyChoices);
|
||||
}
|
||||
|
||||
let name = name.into();
|
||||
let n_choices = choices.len();
|
||||
let distribution = CategoricalDistribution { n_choices };
|
||||
|
||||
// Check if parameter already exists
|
||||
if let Some(existing_dist) = self.distributions.get(&name) {
|
||||
// Verify the distribution matches
|
||||
if let Distribution::Categorical(existing) = existing_dist
|
||||
&& existing.n_choices == n_choices
|
||||
{
|
||||
// Same distribution, return cached value
|
||||
if let Some(ParamValue::Categorical(index)) = self.params.get(&name) {
|
||||
return Ok(choices[*index].clone());
|
||||
}
|
||||
}
|
||||
// Distribution exists but doesn't match
|
||||
return Err(TpeError::ParameterConflict {
|
||||
name,
|
||||
reason: "parameter was previously sampled with different number of choices or type"
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
// Sample using the sampler
|
||||
let dist = Distribution::Categorical(distribution);
|
||||
let index = match self.sample_value(&dist) {
|
||||
ParamValue::Categorical(idx) => idx,
|
||||
_ => unreachable!("Categorical distribution should return Categorical value"),
|
||||
};
|
||||
|
||||
// Store distribution and value (store the index)
|
||||
self.distributions.insert(name.clone(), dist);
|
||||
self.params.insert(name, ParamValue::Categorical(index));
|
||||
|
||||
Ok(choices[index].clone())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
/// The direction of optimization.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
pub enum Direction {
|
||||
/// Minimize the objective value.
|
||||
Minimize,
|
||||
@@ -12,7 +11,6 @@ pub enum Direction {
|
||||
|
||||
/// The state of a trial in its lifecycle.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
pub enum TrialState {
|
||||
/// The trial is currently running.
|
||||
Running,
|
||||
@@ -20,6 +18,4 @@ pub enum TrialState {
|
||||
Complete,
|
||||
/// The trial failed with an error.
|
||||
Failed,
|
||||
/// The trial was pruned (stopped early).
|
||||
Pruned,
|
||||
}
|
||||
|
||||
@@ -1,458 +0,0 @@
|
||||
//! HTML report generation for optimization visualization.
|
||||
//!
|
||||
//! Generates self-contained HTML files with embedded Plotly.js charts
|
||||
//! for offline visualization of optimization results.
|
||||
|
||||
use core::fmt::Write as _;
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::distribution::Distribution;
|
||||
use crate::param::ParamValue;
|
||||
use crate::parameter::ParamId;
|
||||
use crate::sampler::CompletedTrial;
|
||||
use crate::types::{Direction, TrialState};
|
||||
|
||||
/// Generate an HTML report with interactive Plotly.js charts.
|
||||
///
|
||||
/// Creates a self-contained HTML file at `path` containing:
|
||||
/// - **Optimization history**: Objective value vs trial number with best-so-far line
|
||||
/// - **Slice plots**: Objective value vs each parameter (1D scatter)
|
||||
/// - **Parallel coordinates**: Multi-parameter relationship view
|
||||
/// - **Trial timeline**: Duration index of each trial (horizontal bar)
|
||||
/// - **Intermediate values**: Learning curves per trial (if pruning data available)
|
||||
/// - **Parameter importance**: Bar chart (if enough completed trials)
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// Returns an I/O error if the file cannot be created or written.
|
||||
pub fn generate_html_report(
|
||||
study: &crate::Study<f64>,
|
||||
path: impl AsRef<Path>,
|
||||
) -> std::io::Result<()> {
|
||||
let trials = study.trials();
|
||||
let direction = study.direction();
|
||||
let importance = study.param_importance();
|
||||
|
||||
let html = build_html(&trials, direction, &importance);
|
||||
std::fs::write(path, html)
|
||||
}
|
||||
|
||||
fn build_html(
|
||||
trials: &[CompletedTrial<f64>],
|
||||
direction: Direction,
|
||||
importance: &[(String, f64)],
|
||||
) -> String {
|
||||
let mut html = String::with_capacity(8192);
|
||||
|
||||
let dir_label = match direction {
|
||||
Direction::Minimize => "Minimize",
|
||||
Direction::Maximize => "Maximize",
|
||||
};
|
||||
|
||||
// Collect parameter metadata.
|
||||
let param_info = collect_param_info(trials);
|
||||
let has_intermediate = trials.iter().any(|t| !t.intermediate_values.is_empty());
|
||||
|
||||
let _ = write!(
|
||||
html,
|
||||
r#"<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Optimization Report</title>
|
||||
<script src="https://cdn.plot.ly/plotly-2.35.2.min.js"></script>
|
||||
<style>
|
||||
* {{ margin: 0; padding: 0; box-sizing: border-box; }}
|
||||
body {{ font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
|
||||
background: #f5f6fa; color: #2c3e50; padding: 24px; }}
|
||||
h1 {{ text-align: center; margin-bottom: 8px; font-size: 1.8em; }}
|
||||
.subtitle {{ text-align: center; color: #7f8c8d; margin-bottom: 24px; }}
|
||||
.chart {{ background: #fff; border-radius: 8px; box-shadow: 0 2px 8px rgba(0,0,0,0.08);
|
||||
margin-bottom: 24px; padding: 16px; }}
|
||||
.chart-title {{ font-size: 1.1em; font-weight: 600; margin-bottom: 8px; }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Optimization Report</h1>
|
||||
<p class="subtitle">{dir_label} · {n} trials</p>
|
||||
"#,
|
||||
n = trials.len(),
|
||||
);
|
||||
|
||||
// Optimization history chart.
|
||||
html.push_str("<div class=\"chart\"><div class=\"chart-title\">Optimization History</div><div id=\"history\"></div></div>\n");
|
||||
write_history_chart(&mut html, trials, direction);
|
||||
|
||||
// Slice plots.
|
||||
if !param_info.is_empty() {
|
||||
html.push_str("<div class=\"chart\"><div class=\"chart-title\">Slice Plots</div><div id=\"slices\"></div></div>\n");
|
||||
write_slice_charts(&mut html, trials, ¶m_info);
|
||||
}
|
||||
|
||||
// Parallel coordinates.
|
||||
if param_info.len() >= 2 {
|
||||
html.push_str("<div class=\"chart\"><div class=\"chart-title\">Parallel Coordinates</div><div id=\"parcoords\"></div></div>\n");
|
||||
write_parallel_coordinates(&mut html, trials, ¶m_info, direction);
|
||||
}
|
||||
|
||||
// Parameter importance.
|
||||
if !importance.is_empty() {
|
||||
html.push_str("<div class=\"chart\"><div class=\"chart-title\">Parameter Importance</div><div id=\"importance\"></div></div>\n");
|
||||
write_importance_chart(&mut html, importance);
|
||||
}
|
||||
|
||||
// Trial timeline.
|
||||
html.push_str("<div class=\"chart\"><div class=\"chart-title\">Trial Timeline</div><div id=\"timeline\"></div></div>\n");
|
||||
write_timeline_chart(&mut html, trials);
|
||||
|
||||
// Intermediate values.
|
||||
if has_intermediate {
|
||||
html.push_str("<div class=\"chart\"><div class=\"chart-title\">Intermediate Values</div><div id=\"intermediate\"></div></div>\n");
|
||||
write_intermediate_chart(&mut html, trials);
|
||||
}
|
||||
|
||||
html.push_str("</body>\n</html>\n");
|
||||
html
|
||||
}
|
||||
|
||||
/// Metadata about each parameter seen across trials.
|
||||
struct ParamMeta {
|
||||
label: String,
|
||||
#[allow(dead_code)]
|
||||
dist: Option<Distribution>,
|
||||
}
|
||||
|
||||
/// Collect parameter labels and distributions across all trials.
|
||||
fn collect_param_info(trials: &[CompletedTrial<f64>]) -> BTreeMap<ParamId, ParamMeta> {
|
||||
let mut info: BTreeMap<ParamId, ParamMeta> = BTreeMap::new();
|
||||
for trial in trials {
|
||||
for &id in trial.params.keys() {
|
||||
info.entry(id).or_insert_with(|| {
|
||||
let label = trial
|
||||
.param_labels
|
||||
.get(&id)
|
||||
.cloned()
|
||||
.unwrap_or_else(|| id.to_string());
|
||||
let dist = trial.distributions.get(&id).cloned();
|
||||
ParamMeta { label, dist }
|
||||
});
|
||||
}
|
||||
}
|
||||
info
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Chart generators
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
fn write_history_chart(html: &mut String, trials: &[CompletedTrial<f64>], direction: Direction) {
|
||||
let complete: Vec<_> = trials
|
||||
.iter()
|
||||
.filter(|t| t.state == TrialState::Complete)
|
||||
.collect();
|
||||
if complete.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut ids = Vec::with_capacity(complete.len());
|
||||
let mut vals = Vec::with_capacity(complete.len());
|
||||
let mut best_vals = Vec::with_capacity(complete.len());
|
||||
let mut best = complete[0].value;
|
||||
for t in &complete {
|
||||
ids.push(t.id);
|
||||
vals.push(t.value);
|
||||
best = match direction {
|
||||
Direction::Minimize => best.min(t.value),
|
||||
Direction::Maximize => best.max(t.value),
|
||||
};
|
||||
best_vals.push(best);
|
||||
}
|
||||
|
||||
let _ = write!(
|
||||
html,
|
||||
r##"<script>
|
||||
Plotly.newPlot("history", [
|
||||
{{ x: {ids:?}, y: {vals:?}, mode: "markers", name: "Objective", type: "scatter",
|
||||
marker: {{ color: "#3498db", size: 6 }} }},
|
||||
{{ x: {ids:?}, y: {best_vals:?}, mode: "lines", name: "Best so far", type: "scatter",
|
||||
line: {{ color: "#e74c3c", width: 2 }} }}
|
||||
], {{ xaxis: {{ title: "Trial" }}, yaxis: {{ title: "Objective Value" }},
|
||||
margin: {{ t: 10 }}, legend: {{ x: 1, xanchor: "right", y: 1 }} }},
|
||||
{{ responsive: true }});
|
||||
</script>
|
||||
"##,
|
||||
);
|
||||
}
|
||||
|
||||
fn write_slice_charts(
|
||||
html: &mut String,
|
||||
trials: &[CompletedTrial<f64>],
|
||||
param_info: &BTreeMap<ParamId, ParamMeta>,
|
||||
) {
|
||||
let complete: Vec<_> = trials
|
||||
.iter()
|
||||
.filter(|t| t.state == TrialState::Complete)
|
||||
.collect();
|
||||
if complete.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let n_params = param_info.len();
|
||||
let cols = if n_params <= 2 { n_params } else { 2 };
|
||||
let rows = n_params.div_ceil(cols);
|
||||
|
||||
// Build subplot titles and data.
|
||||
let mut subplot_titles = Vec::new();
|
||||
let mut traces = String::new();
|
||||
for (i, (id, meta)) in param_info.iter().enumerate() {
|
||||
subplot_titles.push(format!("\"{}\"", escape_js(&meta.label)));
|
||||
|
||||
let mut x_vals = Vec::new();
|
||||
let mut y_vals = Vec::new();
|
||||
for t in &complete {
|
||||
if let Some(pv) = t.params.get(id) {
|
||||
x_vals.push(param_value_to_f64(pv));
|
||||
y_vals.push(t.value);
|
||||
}
|
||||
}
|
||||
|
||||
let subplot_idx = i + 1;
|
||||
let xa = if subplot_idx == 1 {
|
||||
"x".to_string()
|
||||
} else {
|
||||
format!("x{subplot_idx}")
|
||||
};
|
||||
let ya = if subplot_idx == 1 {
|
||||
"y".to_string()
|
||||
} else {
|
||||
format!("y{subplot_idx}")
|
||||
};
|
||||
let _ = write!(
|
||||
traces,
|
||||
r##"{{ x: {x_vals:?}, y: {y_vals:?}, mode: "markers", type: "scatter",
|
||||
xaxis: "{xa}", yaxis: "{ya}",
|
||||
marker: {{ color: "#3498db", size: 5 }}, showlegend: false }},"##,
|
||||
);
|
||||
}
|
||||
|
||||
let _ = write!(
|
||||
html,
|
||||
r#"<script>
|
||||
Plotly.newPlot("slices", [{traces}],
|
||||
{{ grid: {{ rows: {rows}, columns: {cols}, pattern: "independent" }},
|
||||
annotations: [{annotations}],
|
||||
margin: {{ t: 30 }}, showlegend: false }},
|
||||
{{ responsive: true }});
|
||||
</script>
|
||||
"#,
|
||||
annotations = build_subplot_annotations(&subplot_titles, rows, cols),
|
||||
);
|
||||
}
|
||||
|
||||
fn write_parallel_coordinates(
|
||||
html: &mut String,
|
||||
trials: &[CompletedTrial<f64>],
|
||||
param_info: &BTreeMap<ParamId, ParamMeta>,
|
||||
direction: Direction,
|
||||
) {
|
||||
let complete: Vec<_> = trials
|
||||
.iter()
|
||||
.filter(|t| t.state == TrialState::Complete)
|
||||
.collect();
|
||||
if complete.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut dimensions = String::new();
|
||||
|
||||
// Add objective value as the first dimension.
|
||||
let obj_vals: Vec<f64> = complete.iter().map(|t| t.value).collect();
|
||||
let _ = write!(
|
||||
dimensions,
|
||||
r#"{{ label: "Objective", values: {obj_vals:?} }},"#,
|
||||
);
|
||||
|
||||
// Add each parameter as a dimension.
|
||||
for (id, meta) in param_info {
|
||||
let vals: Vec<f64> = complete
|
||||
.iter()
|
||||
.map(|t| t.params.get(id).map_or(f64::NAN, param_value_to_f64))
|
||||
.collect();
|
||||
let _ = write!(
|
||||
dimensions,
|
||||
r#"{{ label: "{label}", values: {vals:?} }},"#,
|
||||
label = escape_js(&meta.label),
|
||||
);
|
||||
}
|
||||
|
||||
// Color by objective value: green = good, red = bad.
|
||||
let (cmin, cmax) = min_max(&obj_vals);
|
||||
let colorscale = match direction {
|
||||
Direction::Minimize => r##"[[0,"#2ecc71"],[1,"#e74c3c"]]"##,
|
||||
Direction::Maximize => r##"[[0,"#e74c3c"],[1,"#2ecc71"]]"##,
|
||||
};
|
||||
|
||||
let _ = write!(
|
||||
html,
|
||||
r#"<script>
|
||||
Plotly.newPlot("parcoords", [{{
|
||||
type: "parcoords",
|
||||
line: {{ color: {obj_vals:?}, colorscale: {colorscale},
|
||||
cmin: {cmin}, cmax: {cmax}, showscale: true }},
|
||||
dimensions: [{dimensions}]
|
||||
}}], {{ margin: {{ t: 10 }} }}, {{ responsive: true }});
|
||||
</script>
|
||||
"#,
|
||||
);
|
||||
}
|
||||
|
||||
fn write_importance_chart(html: &mut String, importance: &[(String, f64)]) {
|
||||
let names: Vec<_> = importance.iter().map(|(n, _)| format!("\"{n}\"")).collect();
|
||||
let values: Vec<f64> = importance.iter().map(|(_, v)| *v).collect();
|
||||
|
||||
let _ = write!(
|
||||
html,
|
||||
r##"<script>
|
||||
Plotly.newPlot("importance", [{{
|
||||
x: {values:?}, y: [{names}], type: "bar", orientation: "h",
|
||||
marker: {{ color: "#9b59b6" }}
|
||||
}}], {{ xaxis: {{ title: "Importance (|Spearman correlation|)" }},
|
||||
yaxis: {{ automargin: true }}, margin: {{ t: 10, l: 120 }} }},
|
||||
{{ responsive: true }});
|
||||
</script>
|
||||
"##,
|
||||
names = names.join(","),
|
||||
);
|
||||
}
|
||||
|
||||
fn write_timeline_chart(html: &mut String, trials: &[CompletedTrial<f64>]) {
|
||||
let mut ids = Vec::with_capacity(trials.len());
|
||||
let mut colors = Vec::with_capacity(trials.len());
|
||||
let mut labels = Vec::with_capacity(trials.len());
|
||||
|
||||
for t in trials {
|
||||
ids.push(format!("\"Trial {}\"", t.id));
|
||||
let (color, label) = match t.state {
|
||||
TrialState::Complete => ("#2ecc71", "Complete"),
|
||||
TrialState::Pruned => ("#f39c12", "Pruned"),
|
||||
TrialState::Failed => ("#e74c3c", "Failed"),
|
||||
TrialState::Running => ("#3498db", "Running"),
|
||||
};
|
||||
colors.push(format!("\"{color}\""));
|
||||
labels.push(format!("\"{label}\""));
|
||||
}
|
||||
|
||||
// Use trial index as a proxy for duration (no wallclock data available).
|
||||
let indices: Vec<usize> = (0..trials.len()).collect();
|
||||
|
||||
let _ = write!(
|
||||
html,
|
||||
r#"<script>
|
||||
Plotly.newPlot("timeline", [{{
|
||||
y: [{ids}], x: {indices:?}, type: "bar", orientation: "h",
|
||||
text: [{labels}], textposition: "auto",
|
||||
marker: {{ color: [{colors}] }}
|
||||
}}], {{ xaxis: {{ title: "Trial Index" }}, yaxis: {{ automargin: true, autorange: "reversed" }},
|
||||
margin: {{ t: 10, l: 80 }}, showlegend: false }},
|
||||
{{ responsive: true }});
|
||||
</script>
|
||||
"#,
|
||||
ids = ids.join(","),
|
||||
colors = colors.join(","),
|
||||
labels = labels.join(","),
|
||||
);
|
||||
}
|
||||
|
||||
fn write_intermediate_chart(html: &mut String, trials: &[CompletedTrial<f64>]) {
|
||||
let trials_with_iv: Vec<_> = trials
|
||||
.iter()
|
||||
.filter(|t| !t.intermediate_values.is_empty())
|
||||
.collect();
|
||||
if trials_with_iv.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut traces = String::new();
|
||||
for t in &trials_with_iv {
|
||||
let steps: Vec<u64> = t.intermediate_values.iter().map(|(s, _)| *s).collect();
|
||||
let values: Vec<f64> = t.intermediate_values.iter().map(|(_, v)| *v).collect();
|
||||
let color = match t.state {
|
||||
TrialState::Pruned => "#f39c12",
|
||||
_ => "#3498db",
|
||||
};
|
||||
let _ = write!(
|
||||
traces,
|
||||
r#"{{ x: {steps:?}, y: {values:?}, mode: "lines+markers", name: "Trial {id}",
|
||||
line: {{ color: "{color}", width: 1 }}, marker: {{ size: 3 }} }},"#,
|
||||
id = t.id,
|
||||
);
|
||||
}
|
||||
|
||||
let _ = write!(
|
||||
html,
|
||||
r#"<script>
|
||||
Plotly.newPlot("intermediate", [{traces}],
|
||||
{{ xaxis: {{ title: "Step" }}, yaxis: {{ title: "Intermediate Value" }},
|
||||
margin: {{ t: 10 }}, showlegend: true }},
|
||||
{{ responsive: true }});
|
||||
</script>
|
||||
"#,
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
fn param_value_to_f64(pv: &ParamValue) -> f64 {
|
||||
match *pv {
|
||||
ParamValue::Float(v) => v,
|
||||
ParamValue::Int(v) => v as f64,
|
||||
ParamValue::Categorical(v) => v as f64,
|
||||
}
|
||||
}
|
||||
|
||||
fn escape_js(s: &str) -> String {
|
||||
s.replace('\\', "\\\\")
|
||||
.replace('"', "\\\"")
|
||||
.replace('\n', "\\n")
|
||||
}
|
||||
|
||||
fn min_max(vals: &[f64]) -> (f64, f64) {
|
||||
let mut mn = f64::INFINITY;
|
||||
let mut mx = f64::NEG_INFINITY;
|
||||
for &v in vals {
|
||||
if v < mn {
|
||||
mn = v;
|
||||
}
|
||||
if v > mx {
|
||||
mx = v;
|
||||
}
|
||||
}
|
||||
(mn, mx)
|
||||
}
|
||||
|
||||
/// Build Plotly annotation objects to act as subplot titles.
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
fn build_subplot_annotations(titles: &[String], rows: usize, cols: usize) -> String {
|
||||
let mut anns = Vec::new();
|
||||
for (i, title) in titles.iter().enumerate() {
|
||||
let row = i / cols;
|
||||
let col = i % cols;
|
||||
// Compute x/y anchor in paper coordinates.
|
||||
let x = if cols == 1 {
|
||||
0.5
|
||||
} else {
|
||||
(f64::from(u32::try_from(col).unwrap_or(0))) / (cols as f64 - 1.0)
|
||||
};
|
||||
let y = 1.0 - (f64::from(u32::try_from(row).unwrap_or(0))) / (rows as f64).max(1.0) + 0.02;
|
||||
anns.push(format!(
|
||||
r#"{{ text: {title}, x: {x:.3}, y: {y:.3}, xref: "paper", yref: "paper",
|
||||
showarrow: false, font: {{ size: 12 }} }}"#,
|
||||
));
|
||||
}
|
||||
anns.join(",")
|
||||
}
|
||||
+32
-80
@@ -4,25 +4,17 @@
|
||||
|
||||
#![cfg(feature = "async")]
|
||||
|
||||
use optimizer::parameter::{FloatParam, Parameter};
|
||||
use optimizer::sampler::random::RandomSampler;
|
||||
use optimizer::sampler::tpe::TpeSampler;
|
||||
use optimizer::{Direction, Error, Study};
|
||||
use optimizer::{Direction, RandomSampler, Study, TpeError, TpeSampler};
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_optimize_async_basic() {
|
||||
let sampler = RandomSampler::with_seed(42);
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x_param = FloatParam::new(-10.0, 10.0);
|
||||
|
||||
study
|
||||
.optimize_async(10, move |mut trial| {
|
||||
let x_param = x_param.clone();
|
||||
async move {
|
||||
let x = x_param.suggest(&mut trial)?;
|
||||
Ok::<_, Error>((trial, x * x))
|
||||
}
|
||||
.optimize_async(10, |mut trial| async move {
|
||||
let x = trial.suggest_float("x", -10.0, 10.0)?;
|
||||
Ok::<_, TpeError>((trial, x * x))
|
||||
})
|
||||
.await
|
||||
.expect("async optimization should succeed");
|
||||
@@ -33,24 +25,15 @@ async fn test_optimize_async_basic() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_optimize_async_with_tpe() {
|
||||
let sampler = TpeSampler::builder()
|
||||
.seed(42)
|
||||
.n_startup_trials(5)
|
||||
.build()
|
||||
.unwrap();
|
||||
async fn test_optimize_async_with_sampler() {
|
||||
let sampler = TpeSampler::builder().seed(42).n_startup_trials(5).build();
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x_param = FloatParam::new(-5.0, 5.0);
|
||||
|
||||
study
|
||||
.optimize_async(15, move |mut trial| {
|
||||
let x_param = x_param.clone();
|
||||
async move {
|
||||
let x = x_param.suggest(&mut trial)?;
|
||||
Ok::<_, Error>((trial, x * x))
|
||||
}
|
||||
.optimize_async_with_sampler(15, |mut trial| async move {
|
||||
let x = trial.suggest_float("x", -5.0, 5.0)?;
|
||||
Ok::<_, TpeError>((trial, x * x))
|
||||
})
|
||||
.await
|
||||
.expect("async optimization with sampler should succeed");
|
||||
@@ -65,15 +48,10 @@ async fn test_optimize_parallel() {
|
||||
let sampler = RandomSampler::with_seed(42);
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x_param = FloatParam::new(-10.0, 10.0);
|
||||
|
||||
study
|
||||
.optimize_parallel(20, 4, move |mut trial| {
|
||||
let x_param = x_param.clone();
|
||||
async move {
|
||||
let x = x_param.suggest(&mut trial)?;
|
||||
Ok::<_, Error>((trial, x * x))
|
||||
}
|
||||
.optimize_parallel(20, 4, |mut trial| async move {
|
||||
let x = trial.suggest_float("x", -10.0, 10.0)?;
|
||||
Ok::<_, TpeError>((trial, x * x))
|
||||
})
|
||||
.await
|
||||
.expect("parallel optimization should succeed");
|
||||
@@ -82,27 +60,16 @@ async fn test_optimize_parallel() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_optimize_parallel_with_tpe() {
|
||||
let sampler = TpeSampler::builder()
|
||||
.seed(42)
|
||||
.n_startup_trials(5)
|
||||
.build()
|
||||
.unwrap();
|
||||
async fn test_optimize_parallel_with_sampler() {
|
||||
let sampler = TpeSampler::builder().seed(42).n_startup_trials(5).build();
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x_param = FloatParam::new(-5.0, 5.0);
|
||||
let y_param = FloatParam::new(-5.0, 5.0);
|
||||
|
||||
study
|
||||
.optimize_parallel(15, 3, move |mut trial| {
|
||||
let x_param = x_param.clone();
|
||||
let y_param = y_param.clone();
|
||||
async move {
|
||||
let x = x_param.suggest(&mut trial)?;
|
||||
let y = y_param.suggest(&mut trial)?;
|
||||
Ok::<_, Error>((trial, x * x + y * y))
|
||||
}
|
||||
.optimize_parallel_with_sampler(15, 3, |mut trial| async move {
|
||||
let x = trial.suggest_float("x", -5.0, 5.0)?;
|
||||
let y = trial.suggest_float("y", -5.0, 5.0)?;
|
||||
Ok::<_, TpeError>((trial, x * x + y * y))
|
||||
})
|
||||
.await
|
||||
.expect("parallel optimization with sampler should succeed");
|
||||
@@ -122,13 +89,12 @@ async fn test_optimize_async_all_failures() {
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(Error::NoCompletedTrials)),
|
||||
matches!(result, Err(TpeError::NoCompletedTrials)),
|
||||
"should return NoCompletedTrials when all trials fail"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(deprecated)]
|
||||
async fn test_optimize_async_with_sampler_all_failures() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
|
||||
@@ -140,7 +106,7 @@ async fn test_optimize_async_with_sampler_all_failures() {
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(Error::NoCompletedTrials)),
|
||||
matches!(result, Err(TpeError::NoCompletedTrials)),
|
||||
"should return NoCompletedTrials when all trials fail"
|
||||
);
|
||||
}
|
||||
@@ -157,13 +123,12 @@ async fn test_optimize_parallel_all_failures() {
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(Error::NoCompletedTrials)),
|
||||
matches!(result, Err(TpeError::NoCompletedTrials)),
|
||||
"should return NoCompletedTrials when all trials fail"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(deprecated)]
|
||||
async fn test_optimize_parallel_with_sampler_all_failures() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
|
||||
@@ -175,7 +140,7 @@ async fn test_optimize_parallel_with_sampler_all_failures() {
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(Error::NoCompletedTrials)),
|
||||
matches!(result, Err(TpeError::NoCompletedTrials)),
|
||||
"should return NoCompletedTrials when all trials fail"
|
||||
);
|
||||
}
|
||||
@@ -187,18 +152,15 @@ async fn test_optimize_async_partial_failures() {
|
||||
|
||||
let counter = std::sync::atomic::AtomicUsize::new(0);
|
||||
|
||||
let x_param = FloatParam::new(0.0, 10.0);
|
||||
|
||||
study
|
||||
.optimize_async(10, move |mut trial| {
|
||||
.optimize_async(10, |mut trial| {
|
||||
let count = counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
let x_param = x_param.clone();
|
||||
async move {
|
||||
if count.is_multiple_of(2) {
|
||||
let x = x_param.suggest(&mut trial)?;
|
||||
Ok::<_, Error>((trial, x))
|
||||
let x = trial.suggest_float("x", 0.0, 10.0)?;
|
||||
Ok::<_, TpeError>((trial, x))
|
||||
} else {
|
||||
Err(Error::NoCompletedTrials) // Use as error type
|
||||
Err(TpeError::NoCompletedTrials) // Use as error type
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -214,16 +176,11 @@ async fn test_optimize_parallel_high_concurrency() {
|
||||
let sampler = RandomSampler::with_seed(42);
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x_param = FloatParam::new(0.0, 10.0);
|
||||
|
||||
// Run with concurrency higher than n_trials
|
||||
study
|
||||
.optimize_parallel(5, 10, move |mut trial| {
|
||||
let x_param = x_param.clone();
|
||||
async move {
|
||||
let x = x_param.suggest(&mut trial)?;
|
||||
Ok::<_, Error>((trial, x))
|
||||
}
|
||||
.optimize_parallel(5, 10, |mut trial| async move {
|
||||
let x = trial.suggest_float("x", 0.0, 10.0)?;
|
||||
Ok::<_, TpeError>((trial, x))
|
||||
})
|
||||
.await
|
||||
.expect("should handle high concurrency");
|
||||
@@ -236,16 +193,11 @@ async fn test_optimize_parallel_single_concurrency() {
|
||||
let sampler = RandomSampler::with_seed(42);
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x_param = FloatParam::new(0.0, 10.0);
|
||||
|
||||
// Run with concurrency of 1 (sequential)
|
||||
study
|
||||
.optimize_parallel(10, 1, move |mut trial| {
|
||||
let x_param = x_param.clone();
|
||||
async move {
|
||||
let x = x_param.suggest(&mut trial)?;
|
||||
Ok::<_, Error>((trial, x))
|
||||
}
|
||||
.optimize_parallel(10, 1, |mut trial| async move {
|
||||
let x = trial.suggest_float("x", 0.0, 10.0)?;
|
||||
Ok::<_, TpeError>((trial, x))
|
||||
})
|
||||
.await
|
||||
.expect("should work with single concurrency");
|
||||
|
||||
@@ -1,133 +0,0 @@
|
||||
//! Integration tests for the BOHB sampler.
|
||||
|
||||
#![allow(
|
||||
clippy::cast_sign_loss,
|
||||
clippy::cast_precision_loss,
|
||||
clippy::cast_possible_truncation
|
||||
)]
|
||||
|
||||
use optimizer::parameter::{FloatParam, Parameter};
|
||||
use optimizer::sampler::bohb::BohbSampler;
|
||||
use optimizer::{Direction, Error, Study, TrialPruned};
|
||||
|
||||
#[test]
|
||||
fn bohb_converges_on_quadratic() {
|
||||
let bohb = BohbSampler::builder()
|
||||
.min_resource(1)
|
||||
.max_resource(9)
|
||||
.reduction_factor(3)
|
||||
.min_points_in_model(5)
|
||||
.seed(42)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let pruner = bohb.matching_pruner(Direction::Minimize);
|
||||
let study: Study<f64> = Study::with_sampler_and_pruner(Direction::Minimize, bohb, pruner);
|
||||
|
||||
let x_param = FloatParam::new(-10.0, 10.0);
|
||||
|
||||
study
|
||||
.optimize(60, |trial| {
|
||||
let x = x_param.suggest(trial)?;
|
||||
|
||||
// Report intermediate values at budget steps 1, 3, 9
|
||||
let obj = (x - 3.0).powi(2);
|
||||
// Simulate budget-based evaluation with noise decreasing at higher budgets
|
||||
trial.report(1, obj + 5.0);
|
||||
trial.report(3, obj + 1.0);
|
||||
trial.report(9, obj);
|
||||
|
||||
Ok::<_, Error>(obj)
|
||||
})
|
||||
.expect("optimization should succeed");
|
||||
|
||||
let best = study.best_trial().expect("should have trials");
|
||||
assert!(
|
||||
best.value < 10.0,
|
||||
"BOHB should find a reasonable solution, got {}",
|
||||
best.value
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bohb_with_pruning() {
|
||||
let bohb = BohbSampler::builder()
|
||||
.min_resource(1)
|
||||
.max_resource(27)
|
||||
.reduction_factor(3)
|
||||
.min_points_in_model(3)
|
||||
.seed(123)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let pruner = bohb.matching_pruner(Direction::Minimize);
|
||||
let study: Study<f64> = Study::with_sampler_and_pruner(Direction::Minimize, bohb, pruner);
|
||||
|
||||
let x_param = FloatParam::new(-5.0, 5.0);
|
||||
|
||||
study
|
||||
.optimize(40, |trial| {
|
||||
let x = x_param.suggest(trial)?;
|
||||
let obj = x * x;
|
||||
|
||||
// Report at each rung step and check for pruning
|
||||
for &step in &[1u64, 3, 9, 27] {
|
||||
let noisy_obj = obj + 10.0 / step as f64;
|
||||
trial.report(step, noisy_obj);
|
||||
|
||||
if trial.should_prune() {
|
||||
return Err(TrialPruned.into());
|
||||
}
|
||||
}
|
||||
|
||||
Ok::<_, Error>(obj)
|
||||
})
|
||||
.expect("optimization should succeed");
|
||||
|
||||
// Verify we have completed trials
|
||||
let best = study.best_trial().expect("should have at least one trial");
|
||||
assert!(
|
||||
best.value < 25.0,
|
||||
"best value {} should be reasonable",
|
||||
best.value
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bohb_uses_budget_conditioned_history() {
|
||||
// Verify that BOHB conditions on budget level by testing that samples
|
||||
// are influenced by intermediate values, not just final values.
|
||||
let bohb = BohbSampler::builder()
|
||||
.min_resource(1)
|
||||
.max_resource(9)
|
||||
.reduction_factor(3)
|
||||
.min_points_in_model(3)
|
||||
.seed(42)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let pruner = bohb.matching_pruner(Direction::Minimize);
|
||||
let study: Study<f64> = Study::with_sampler_and_pruner(Direction::Minimize, bohb, pruner);
|
||||
|
||||
let x_param = FloatParam::new(0.0, 10.0);
|
||||
|
||||
study
|
||||
.optimize(30, |trial| {
|
||||
let x = x_param.suggest(trial)?;
|
||||
// Intermediate values that guide optimization toward x=2
|
||||
trial.report(1, (x - 2.0).powi(2) + 1.0);
|
||||
trial.report(3, (x - 2.0).powi(2) + 0.5);
|
||||
trial.report(9, (x - 2.0).powi(2));
|
||||
|
||||
Ok::<_, Error>((x - 2.0).powi(2))
|
||||
})
|
||||
.expect("optimization should succeed");
|
||||
|
||||
let best = study.best_trial().unwrap();
|
||||
let best_x: f64 = best.get(&x_param).unwrap();
|
||||
// Should find x reasonably close to 2.0
|
||||
assert!(
|
||||
(best_x - 2.0).abs() < 5.0,
|
||||
"BOHB should explore near x=2, got x={best_x}"
|
||||
);
|
||||
}
|
||||
@@ -1,259 +0,0 @@
|
||||
#![cfg(feature = "cma-es")]
|
||||
|
||||
use optimizer::prelude::*;
|
||||
use optimizer::sampler::cma_es::CmaEsSampler;
|
||||
|
||||
#[test]
|
||||
fn sphere_function() {
|
||||
let sampler = CmaEsSampler::with_seed(42);
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x = FloatParam::new(-5.0, 5.0).name("x");
|
||||
let y = FloatParam::new(-5.0, 5.0).name("y");
|
||||
|
||||
study
|
||||
.optimize(200, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
let yv = y.suggest(trial)?;
|
||||
Ok::<_, Error>(xv * xv + yv * yv)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let best = study.best_trial().unwrap();
|
||||
assert!(
|
||||
best.value < 1.0,
|
||||
"sphere best value should be < 1.0, got {}",
|
||||
best.value
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rosenbrock_function() {
|
||||
let sampler = CmaEsSampler::builder().population_size(20).seed(42).build();
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x = FloatParam::new(-5.0, 5.0).name("x");
|
||||
let y = FloatParam::new(-5.0, 5.0).name("y");
|
||||
|
||||
study
|
||||
.optimize(300, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
let yv = y.suggest(trial)?;
|
||||
let val = (1.0 - xv).powi(2) + 100.0 * (yv - xv * xv).powi(2);
|
||||
Ok::<_, Error>(val)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let best = study.best_trial().unwrap();
|
||||
// Rosenbrock minimum is 0 at (1, 1); we just check reasonable convergence
|
||||
assert!(
|
||||
best.value < 50.0,
|
||||
"rosenbrock best value should be < 50.0, got {}",
|
||||
best.value
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bounds_respected() {
|
||||
let sampler = CmaEsSampler::with_seed(123);
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x = FloatParam::new(-2.0, 3.0).name("x");
|
||||
let y = FloatParam::new(0.0, 10.0).name("y");
|
||||
|
||||
study
|
||||
.optimize(100, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
let yv = y.suggest(trial)?;
|
||||
Ok::<_, Error>(xv + yv)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
for trial in study.trials() {
|
||||
let xv: f64 = trial.get(&x).unwrap();
|
||||
let yv: f64 = trial.get(&y).unwrap();
|
||||
assert!((-2.0..=3.0).contains(&xv), "x = {xv} out of bounds [-2, 3]");
|
||||
assert!((0.0..=10.0).contains(&yv), "y = {yv} out of bounds [0, 10]");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_params_float_and_categorical() {
|
||||
let sampler = CmaEsSampler::with_seed(42);
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x = FloatParam::new(-5.0, 5.0).name("x");
|
||||
let cat = CategoricalParam::new(vec!["a", "b", "c"]).name("cat");
|
||||
|
||||
study
|
||||
.optimize(50, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
let cv = cat.suggest(trial)?;
|
||||
let penalty = match cv {
|
||||
"a" => 0.0,
|
||||
"b" => 1.0,
|
||||
_ => 2.0,
|
||||
};
|
||||
Ok::<_, Error>(xv * xv + penalty)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let best = study.best_trial().unwrap();
|
||||
// Should find a reasonable value
|
||||
assert!(
|
||||
best.value < 10.0,
|
||||
"best value should be < 10.0, got {}",
|
||||
best.value
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seeded_reproducibility() {
|
||||
let x = FloatParam::new(-5.0, 5.0).name("x");
|
||||
let y = FloatParam::new(-5.0, 5.0).name("y");
|
||||
|
||||
let run = |seed: u64| {
|
||||
let sampler = CmaEsSampler::with_seed(seed);
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
study
|
||||
.optimize(50, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
let yv = y.suggest(trial)?;
|
||||
Ok::<_, Error>(xv * xv + yv * yv)
|
||||
})
|
||||
.unwrap();
|
||||
study.trials().iter().map(|t| t.value).collect::<Vec<_>>()
|
||||
};
|
||||
|
||||
let results1 = run(42);
|
||||
let results2 = run(42);
|
||||
assert_eq!(results1, results2, "same seed should produce same results");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn different_seeds_different_results() {
|
||||
let x = FloatParam::new(-5.0, 5.0).name("x");
|
||||
let y = FloatParam::new(-5.0, 5.0).name("y");
|
||||
|
||||
let run = |seed: u64| {
|
||||
let sampler = CmaEsSampler::with_seed(seed);
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
study
|
||||
.optimize(20, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
let yv = y.suggest(trial)?;
|
||||
Ok::<_, Error>(xv * xv + yv * yv)
|
||||
})
|
||||
.unwrap();
|
||||
study.trials().iter().map(|t| t.value).collect::<Vec<_>>()
|
||||
};
|
||||
|
||||
let results1 = run(42);
|
||||
let results2 = run(99);
|
||||
assert_ne!(
|
||||
results1, results2,
|
||||
"different seeds should produce different results"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_dimension() {
|
||||
let sampler = CmaEsSampler::with_seed(42);
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x = FloatParam::new(-10.0, 10.0).name("x");
|
||||
|
||||
study
|
||||
.optimize(100, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
Ok::<_, Error>((xv - 3.0).powi(2))
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let best = study.best_trial().unwrap();
|
||||
assert!(
|
||||
best.value < 1.0,
|
||||
"1-D optimization should converge, got {}",
|
||||
best.value
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn integer_params() {
|
||||
let sampler = CmaEsSampler::with_seed(42);
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let n = IntParam::new(1, 20).name("n");
|
||||
|
||||
study
|
||||
.optimize(100, |trial| {
|
||||
let nv = n.suggest(trial)?;
|
||||
// Minimum at n = 10
|
||||
Ok::<_, Error>(((nv - 10) * (nv - 10)) as f64)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let best = study.best_trial().unwrap();
|
||||
let best_n: i64 = best.get(&n).unwrap();
|
||||
assert!(
|
||||
(1..=20).contains(&best_n),
|
||||
"integer value {best_n} out of bounds"
|
||||
);
|
||||
assert!(
|
||||
best.value < 10.0,
|
||||
"integer optimization should converge, got {}",
|
||||
best.value
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn log_scale_params() {
|
||||
let sampler = CmaEsSampler::with_seed(42);
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let lr = FloatParam::new(1e-5, 1.0).log_scale().name("lr");
|
||||
|
||||
study
|
||||
.optimize(100, |trial| {
|
||||
let lrv = lr.suggest(trial)?;
|
||||
// Minimum at lr = 0.01
|
||||
Ok::<_, Error>((lrv.ln() - 0.01_f64.ln()).powi(2))
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
for trial in study.trials() {
|
||||
let lrv: f64 = trial.get(&lr).unwrap();
|
||||
assert!(
|
||||
(1e-5..=1.0).contains(&lrv),
|
||||
"log-scale value {lrv} out of bounds"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_population_size_and_sigma() {
|
||||
let sampler = CmaEsSampler::builder()
|
||||
.sigma0(1.0)
|
||||
.population_size(10)
|
||||
.seed(42)
|
||||
.build();
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x = FloatParam::new(-5.0, 5.0).name("x");
|
||||
let y = FloatParam::new(-5.0, 5.0).name("y");
|
||||
|
||||
study
|
||||
.optimize(100, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
let yv = y.suggest(trial)?;
|
||||
Ok::<_, Error>(xv * xv + yv * yv)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let best = study.best_trial().unwrap();
|
||||
assert!(
|
||||
best.value < 5.0,
|
||||
"custom config optimization should work, got {}",
|
||||
best.value
|
||||
);
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
use optimizer::Trial;
|
||||
use optimizer::parameter::{EnumParam, Parameter};
|
||||
use optimizer_derive::Categorical;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Categorical)]
|
||||
enum Color {
|
||||
Red,
|
||||
Green,
|
||||
Blue,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Categorical)]
|
||||
enum SingleVariant {
|
||||
Only,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derive_categorical_n_choices() {
|
||||
use optimizer::parameter::Categorical;
|
||||
assert_eq!(Color::N_CHOICES, 3);
|
||||
assert_eq!(SingleVariant::N_CHOICES, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derive_categorical_roundtrip() {
|
||||
use optimizer::parameter::Categorical;
|
||||
for i in 0..Color::N_CHOICES {
|
||||
let val = Color::from_index(i);
|
||||
assert_eq!(val.to_index(), i);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derive_categorical_values() {
|
||||
use optimizer::parameter::Categorical;
|
||||
assert_eq!(Color::from_index(0), Color::Red);
|
||||
assert_eq!(Color::from_index(1), Color::Green);
|
||||
assert_eq!(Color::from_index(2), Color::Blue);
|
||||
assert_eq!(Color::Red.to_index(), 0);
|
||||
assert_eq!(Color::Green.to_index(), 1);
|
||||
assert_eq!(Color::Blue.to_index(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derive_categorical_with_enum_param() {
|
||||
let mut trial = Trial::new(0);
|
||||
let param = EnumParam::<Color>::new();
|
||||
let color = param.suggest(&mut trial).unwrap();
|
||||
assert!([Color::Red, Color::Green, Color::Blue].contains(&color));
|
||||
|
||||
// Cached (same param id)
|
||||
let color2 = param.suggest(&mut trial).unwrap();
|
||||
assert_eq!(color, color2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derive_categorical_suggest_via_trial() {
|
||||
let mut trial = Trial::new(0);
|
||||
let color = trial.suggest_param(&EnumParam::<Color>::new()).unwrap();
|
||||
assert!([Color::Red, Color::Green, Color::Blue].contains(&color));
|
||||
}
|
||||
@@ -1,230 +0,0 @@
|
||||
use optimizer::parameter::{FloatParam, IntParam, Parameter};
|
||||
use optimizer::sampler::random::RandomSampler;
|
||||
use optimizer::{Direction, Study};
|
||||
|
||||
#[test]
|
||||
fn csv_empty_study_produces_header_only() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let mut buf = Vec::new();
|
||||
study.to_csv(&mut buf).unwrap();
|
||||
let csv = String::from_utf8(buf).unwrap();
|
||||
assert_eq!(csv, "trial_id,value,state\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_includes_all_trial_data() {
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(42));
|
||||
let x = FloatParam::new(0.0, 10.0).name("x");
|
||||
let y = IntParam::new(1, 5).name("y");
|
||||
|
||||
study
|
||||
.optimize(3, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
let yv = y.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(xv + yv as f64)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let mut buf = Vec::new();
|
||||
study.to_csv(&mut buf).unwrap();
|
||||
let csv = String::from_utf8(buf).unwrap();
|
||||
|
||||
let lines: Vec<&str> = csv.lines().collect();
|
||||
// Header + 3 data rows.
|
||||
assert_eq!(lines.len(), 4);
|
||||
|
||||
// Header should contain our parameter names.
|
||||
let header = lines[0];
|
||||
assert!(header.starts_with("trial_id,value,state"));
|
||||
assert!(header.contains("x"));
|
||||
assert!(header.contains("y"));
|
||||
|
||||
// Each data row should have the right number of columns.
|
||||
let n_cols = header.split(',').count();
|
||||
for line in &lines[1..] {
|
||||
assert_eq!(line.split(',').count(), n_cols);
|
||||
}
|
||||
|
||||
// All rows should have "Complete" state.
|
||||
for line in &lines[1..] {
|
||||
assert!(line.contains("Complete"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_handles_pruned_trials() {
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(42));
|
||||
let x = FloatParam::new(0.0, 10.0).name("x");
|
||||
|
||||
// First trial: complete
|
||||
let mut trial = study.create_trial();
|
||||
let _ = x.suggest(&mut trial).unwrap();
|
||||
study.complete_trial(trial, 1.0);
|
||||
|
||||
// Second trial: pruned
|
||||
let mut trial = study.create_trial();
|
||||
let _ = x.suggest(&mut trial).unwrap();
|
||||
study.prune_trial(trial);
|
||||
|
||||
let mut buf = Vec::new();
|
||||
study.to_csv(&mut buf).unwrap();
|
||||
let csv = String::from_utf8(buf).unwrap();
|
||||
|
||||
let lines: Vec<&str> = csv.lines().collect();
|
||||
assert_eq!(lines.len(), 3); // header + 2 data rows
|
||||
|
||||
// Pruned trial should have empty value.
|
||||
let pruned_line = lines[2];
|
||||
assert!(pruned_line.contains("Pruned"));
|
||||
// The value field (second column) should be empty.
|
||||
let cols: Vec<&str> = pruned_line.split(',').collect();
|
||||
assert_eq!(cols[2], "Pruned");
|
||||
assert_eq!(cols[1], ""); // empty value for pruned
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_handles_different_parameter_sets() {
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(42));
|
||||
let x = FloatParam::new(0.0, 10.0).name("x");
|
||||
let y = FloatParam::new(0.0, 10.0).name("y");
|
||||
|
||||
// First trial: only x
|
||||
let mut trial = study.create_trial();
|
||||
let xv = x.suggest(&mut trial).unwrap();
|
||||
study.complete_trial(trial, xv);
|
||||
|
||||
// Second trial: only y
|
||||
let mut trial = study.create_trial();
|
||||
let yv = y.suggest(&mut trial).unwrap();
|
||||
study.complete_trial(trial, yv);
|
||||
|
||||
let mut buf = Vec::new();
|
||||
study.to_csv(&mut buf).unwrap();
|
||||
let csv = String::from_utf8(buf).unwrap();
|
||||
|
||||
let lines: Vec<&str> = csv.lines().collect();
|
||||
assert_eq!(lines.len(), 3);
|
||||
|
||||
// Both x and y columns should exist.
|
||||
let header = lines[0];
|
||||
assert!(header.contains("x"));
|
||||
assert!(header.contains("y"));
|
||||
|
||||
// Each row has the right column count (missing params are empty).
|
||||
let n_cols = header.split(',').count();
|
||||
for line in &lines[1..] {
|
||||
assert_eq!(line.split(',').count(), n_cols);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_output_is_parseable() {
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(42));
|
||||
let lr = FloatParam::new(0.001, 0.1).name("learning_rate");
|
||||
let layers = IntParam::new(1, 5).name("n_layers");
|
||||
|
||||
study
|
||||
.optimize(5, |trial| {
|
||||
let l = lr.suggest(trial)?;
|
||||
let n = layers.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(l * n as f64)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let mut buf = Vec::new();
|
||||
study.to_csv(&mut buf).unwrap();
|
||||
let csv = String::from_utf8(buf).unwrap();
|
||||
|
||||
// Parse each row: every value field should be a valid f64 for complete trials.
|
||||
let lines: Vec<&str> = csv.lines().collect();
|
||||
for line in &lines[1..] {
|
||||
let cols: Vec<&str> = line.split(',').collect();
|
||||
// trial_id should be a number
|
||||
cols[0].parse::<u64>().unwrap();
|
||||
// value should be parseable as f64
|
||||
cols[1].parse::<f64>().unwrap();
|
||||
// state should be a known value
|
||||
assert!(["Complete", "Pruned", "Failed", "Running"].contains(&cols[2]));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_csv_writes_file() {
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(42));
|
||||
let x = FloatParam::new(0.0, 10.0).name("x");
|
||||
study
|
||||
.optimize(3, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(xv * xv)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let dir = std::env::temp_dir().join("optimizer_export_test");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("test_export.csv");
|
||||
|
||||
study.export_csv(&path).unwrap();
|
||||
|
||||
let contents = std::fs::read_to_string(&path).unwrap();
|
||||
assert!(contents.starts_with("trial_id,value,state"));
|
||||
assert!(contents.lines().count() == 4); // header + 3 rows
|
||||
|
||||
// Clean up.
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[cfg(feature = "serde")]
|
||||
#[test]
|
||||
fn export_json_writes_file() {
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(42));
|
||||
let x = FloatParam::new(0.0, 10.0).name("x");
|
||||
study
|
||||
.optimize(3, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(xv * xv)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let dir = std::env::temp_dir().join("optimizer_json_export_test");
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("test_export.json");
|
||||
|
||||
study.export_json(&path).unwrap();
|
||||
|
||||
let contents = std::fs::read_to_string(&path).unwrap();
|
||||
let parsed: serde_json::Value = serde_json::from_str(&contents).unwrap();
|
||||
let arr = parsed.as_array().unwrap();
|
||||
assert_eq!(arr.len(), 3);
|
||||
|
||||
// Each entry should have the expected fields.
|
||||
for entry in arr {
|
||||
assert!(entry.get("id").is_some());
|
||||
assert!(entry.get("value").is_some());
|
||||
assert!(entry.get("state").is_some());
|
||||
assert!(entry.get("params").is_some());
|
||||
}
|
||||
|
||||
// Clean up.
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csv_includes_user_attributes() {
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(42));
|
||||
let x = FloatParam::new(0.0, 10.0).name("x");
|
||||
|
||||
study
|
||||
.optimize(2, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
trial.set_user_attr("training_time_secs", 45.2);
|
||||
Ok::<_, optimizer::Error>(xv * xv)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let mut buf = Vec::new();
|
||||
study.to_csv(&mut buf).unwrap();
|
||||
let csv = String::from_utf8(buf).unwrap();
|
||||
|
||||
let header = csv.lines().next().unwrap();
|
||||
assert!(header.contains("training_time_secs"));
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
//! Integration tests for fANOVA parameter importance.
|
||||
|
||||
#![cfg(feature = "fanova")]
|
||||
|
||||
use optimizer::prelude::*;
|
||||
|
||||
#[test]
|
||||
fn fanova_dominant_parameter() {
|
||||
// f(x, y) = x^2 — x should dominate
|
||||
let x = FloatParam::new(0.0, 10.0).name("x");
|
||||
let y = FloatParam::new(0.0, 10.0).name("y");
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(42));
|
||||
study
|
||||
.optimize(50, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
let _yv = y.suggest(trial)?;
|
||||
Ok::<_, Error>(xv * xv)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let result = study.fanova().unwrap();
|
||||
assert_eq!(result.main_effects[0].0, "x");
|
||||
assert!(
|
||||
result.main_effects[0].1 > 0.7,
|
||||
"x importance = {}",
|
||||
result.main_effects[0].1
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fanova_interaction() {
|
||||
// f(x, y) = x * y — both matter and interact
|
||||
let x = FloatParam::new(0.0, 10.0).name("x");
|
||||
let y = FloatParam::new(0.0, 10.0).name("y");
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(7));
|
||||
study
|
||||
.optimize(100, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
let yv = y.suggest(trial)?;
|
||||
Ok::<_, Error>(xv * yv)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let config = FanovaConfig {
|
||||
n_trees: 128,
|
||||
..FanovaConfig::default()
|
||||
};
|
||||
let result = study.fanova_with_config(&config).unwrap();
|
||||
|
||||
// Should detect interaction
|
||||
assert!(
|
||||
!result.interactions.is_empty(),
|
||||
"should detect x*y interaction"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fanova_consistent_with_correlation() {
|
||||
// f(x, y) = 3*x + 0.5*y — x should rank higher in both methods
|
||||
let x = FloatParam::new(0.0, 10.0).name("x");
|
||||
let y = FloatParam::new(0.0, 10.0).name("y");
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(99));
|
||||
study
|
||||
.optimize(80, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
let yv = y.suggest(trial)?;
|
||||
Ok::<_, Error>(3.0 * xv + 0.5 * yv)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let corr = study.param_importance();
|
||||
let fanova = study.fanova().unwrap();
|
||||
|
||||
// Both methods should rank x above y
|
||||
assert_eq!(corr[0].0, "x", "correlation should rank x first");
|
||||
assert_eq!(fanova.main_effects[0].0, "x", "fanova should rank x first");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fanova_too_few_trials() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
|
||||
let result = study.fanova();
|
||||
assert!(result.is_err(), "should error with no trials");
|
||||
}
|
||||
@@ -1,166 +0,0 @@
|
||||
use optimizer::parameter::{CategoricalParam, FloatParam, IntParam, Parameter};
|
||||
use optimizer::{Direction, Study};
|
||||
|
||||
#[test]
|
||||
fn known_perfect_correlation() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let x = FloatParam::new(0.0, 100.0).name("x");
|
||||
|
||||
// Objective = x, so perfect correlation.
|
||||
for _ in 0..30 {
|
||||
let mut trial = study.ask();
|
||||
let xv = x.suggest(&mut trial).unwrap();
|
||||
study.tell(trial, Ok::<_, &str>(xv));
|
||||
}
|
||||
|
||||
let importance = study.param_importance();
|
||||
assert_eq!(importance.len(), 1);
|
||||
assert_eq!(importance[0].0, "x");
|
||||
assert!(
|
||||
(importance[0].1 - 1.0).abs() < 1e-10,
|
||||
"single param should be 1.0"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_effect_parameter() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let x = FloatParam::new(0.0, 100.0).name("x");
|
||||
let noise = FloatParam::new(0.0, 100.0).name("noise");
|
||||
|
||||
// Objective depends only on x; noise is unused in objective.
|
||||
for _ in 0..50 {
|
||||
let mut trial = study.ask();
|
||||
let xv = x.suggest(&mut trial).unwrap();
|
||||
let _nv = noise.suggest(&mut trial).unwrap();
|
||||
study.tell(trial, Ok::<_, &str>(xv));
|
||||
}
|
||||
|
||||
let importance = study.param_importance();
|
||||
assert_eq!(importance.len(), 2);
|
||||
// x should have much higher importance than noise.
|
||||
let x_score = importance.iter().find(|(l, _)| l == "x").unwrap().1;
|
||||
let noise_score = importance.iter().find(|(l, _)| l == "noise").unwrap().1;
|
||||
assert!(
|
||||
x_score > noise_score,
|
||||
"x ({x_score}) should outrank noise ({noise_score})"
|
||||
);
|
||||
// x should dominate
|
||||
assert!(x_score > 0.7, "x importance {x_score} should be dominant");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiple_parameters_varying_importance() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let x = FloatParam::new(0.0, 10.0).name("x");
|
||||
let y = FloatParam::new(0.0, 10.0).name("y");
|
||||
|
||||
// Objective = 10*x + 0.01*y → x should be far more important.
|
||||
for _ in 0..50 {
|
||||
let mut trial = study.ask();
|
||||
let xv = x.suggest(&mut trial).unwrap();
|
||||
let yv = y.suggest(&mut trial).unwrap();
|
||||
study.tell(trial, Ok::<_, &str>(10.0 * xv + 0.01 * yv));
|
||||
}
|
||||
|
||||
let importance = study.param_importance();
|
||||
assert_eq!(importance.len(), 2);
|
||||
assert_eq!(importance[0].0, "x", "x should rank first");
|
||||
assert!(importance[0].1 > importance[1].1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fewer_than_two_trials_returns_empty() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
|
||||
// 0 trials
|
||||
assert!(study.param_importance().is_empty());
|
||||
|
||||
// 1 trial
|
||||
let x = FloatParam::new(0.0, 1.0).name("x");
|
||||
let mut trial = study.ask();
|
||||
let xv = x.suggest(&mut trial).unwrap();
|
||||
study.tell(trial, Ok::<_, &str>(xv));
|
||||
assert!(study.param_importance().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn int_parameter() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let n = IntParam::new(1, 100).name("n");
|
||||
|
||||
for _ in 0..30 {
|
||||
let mut trial = study.ask();
|
||||
let nv = n.suggest(&mut trial).unwrap();
|
||||
study.tell(trial, Ok::<_, &str>(nv as f64));
|
||||
}
|
||||
|
||||
let importance = study.param_importance();
|
||||
assert_eq!(importance.len(), 1);
|
||||
assert_eq!(importance[0].0, "n");
|
||||
assert!(importance[0].1 > 0.9);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn categorical_parameter() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let cat = CategoricalParam::new(vec!["a", "b", "c"]).name("cat");
|
||||
let x = FloatParam::new(0.0, 100.0).name("x");
|
||||
|
||||
// Objective depends only on x; categorical is random noise.
|
||||
for _ in 0..50 {
|
||||
let mut trial = study.ask();
|
||||
let _c = cat.suggest(&mut trial).unwrap();
|
||||
let xv = x.suggest(&mut trial).unwrap();
|
||||
study.tell(trial, Ok::<_, &str>(xv));
|
||||
}
|
||||
|
||||
let importance = study.param_importance();
|
||||
assert_eq!(importance.len(), 2);
|
||||
let x_score = importance.iter().find(|(l, _)| l == "x").unwrap().1;
|
||||
assert!(x_score > 0.5, "x should dominate over categorical noise");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalization_sums_to_one() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let x = FloatParam::new(0.0, 10.0).name("x");
|
||||
let y = FloatParam::new(0.0, 10.0).name("y");
|
||||
let z = FloatParam::new(0.0, 10.0).name("z");
|
||||
|
||||
for _ in 0..50 {
|
||||
let mut trial = study.ask();
|
||||
let xv = x.suggest(&mut trial).unwrap();
|
||||
let yv = y.suggest(&mut trial).unwrap();
|
||||
let zv = z.suggest(&mut trial).unwrap();
|
||||
study.tell(trial, Ok::<_, &str>(xv + 0.5 * yv + 0.1 * zv));
|
||||
}
|
||||
|
||||
let importance = study.param_importance();
|
||||
let sum: f64 = importance.iter().map(|(_, s)| *s).sum();
|
||||
assert!(
|
||||
(sum - 1.0).abs() < 1e-10,
|
||||
"scores should sum to 1.0, got {sum}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn label_when_unnamed_uses_debug() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
// No .name() call → label defaults to Debug representation.
|
||||
let x = FloatParam::new(0.0, 10.0);
|
||||
|
||||
for _ in 0..10 {
|
||||
let mut trial = study.ask();
|
||||
let xv = x.suggest(&mut trial).unwrap();
|
||||
study.tell(trial, Ok::<_, &str>(xv));
|
||||
}
|
||||
|
||||
let importance = study.param_importance();
|
||||
assert_eq!(importance.len(), 1);
|
||||
assert!(
|
||||
importance[0].0.starts_with("FloatParam"),
|
||||
"expected Debug label, got {:?}",
|
||||
importance[0].0
|
||||
);
|
||||
}
|
||||
+308
-1411
File diff suppressed because it is too large
Load Diff
@@ -1,228 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use optimizer::Direction;
|
||||
use optimizer::pruner::{MedianPruner, Pruner};
|
||||
use optimizer::sampler::CompletedTrial;
|
||||
|
||||
/// Helper to build a completed trial with given intermediate values.
|
||||
fn trial_with_values(id: u64, intermediate_values: Vec<(u64, f64)>) -> CompletedTrial {
|
||||
CompletedTrial::with_intermediate_values(
|
||||
id,
|
||||
HashMap::new(),
|
||||
HashMap::new(),
|
||||
HashMap::new(),
|
||||
0.0,
|
||||
intermediate_values,
|
||||
HashMap::new(),
|
||||
)
|
||||
}
|
||||
|
||||
// --- Minimize direction ---
|
||||
|
||||
#[test]
|
||||
fn prune_when_worse_than_median_minimize() {
|
||||
let pruner = MedianPruner::new(Direction::Minimize);
|
||||
// 3 completed trials with values at step 2: [1.0, 2.0, 3.0] => median = 2.0
|
||||
let completed = vec![
|
||||
trial_with_values(0, vec![(0, 0.5), (1, 0.8), (2, 1.0)]),
|
||||
trial_with_values(1, vec![(0, 0.6), (1, 1.5), (2, 2.0)]),
|
||||
trial_with_values(2, vec![(0, 0.7), (1, 2.0), (2, 3.0)]),
|
||||
];
|
||||
// Current trial value at step 2 is 2.5 > median 2.0 => prune
|
||||
let current = vec![(0, 0.5), (1, 1.0), (2, 2.5)];
|
||||
assert!(pruner.should_prune(3, 2, ¤t, &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_prune_when_better_than_median_minimize() {
|
||||
let pruner = MedianPruner::new(Direction::Minimize);
|
||||
let completed = vec![
|
||||
trial_with_values(0, vec![(0, 0.5), (1, 0.8), (2, 1.0)]),
|
||||
trial_with_values(1, vec![(0, 0.6), (1, 1.5), (2, 2.0)]),
|
||||
trial_with_values(2, vec![(0, 0.7), (1, 2.0), (2, 3.0)]),
|
||||
];
|
||||
// Current trial value at step 2 is 1.5 < median 2.0 => don't prune
|
||||
let current = vec![(0, 0.5), (1, 1.0), (2, 1.5)];
|
||||
assert!(!pruner.should_prune(3, 2, ¤t, &completed));
|
||||
}
|
||||
|
||||
// --- Maximize direction ---
|
||||
|
||||
#[test]
|
||||
fn prune_when_worse_than_median_maximize() {
|
||||
let pruner = MedianPruner::new(Direction::Maximize);
|
||||
// Values at step 1: [5.0, 7.0, 9.0] => median = 7.0
|
||||
let completed = vec![
|
||||
trial_with_values(0, vec![(0, 3.0), (1, 5.0)]),
|
||||
trial_with_values(1, vec![(0, 4.0), (1, 7.0)]),
|
||||
trial_with_values(2, vec![(0, 5.0), (1, 9.0)]),
|
||||
];
|
||||
// Current value 6.0 < median 7.0 => prune (worse for maximize)
|
||||
let current = vec![(0, 4.0), (1, 6.0)];
|
||||
assert!(pruner.should_prune(3, 1, ¤t, &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_prune_when_better_than_median_maximize() {
|
||||
let pruner = MedianPruner::new(Direction::Maximize);
|
||||
let completed = vec![
|
||||
trial_with_values(0, vec![(0, 3.0), (1, 5.0)]),
|
||||
trial_with_values(1, vec![(0, 4.0), (1, 7.0)]),
|
||||
trial_with_values(2, vec![(0, 5.0), (1, 9.0)]),
|
||||
];
|
||||
// Current value 8.0 > median 7.0 => don't prune
|
||||
let current = vec![(0, 4.0), (1, 8.0)];
|
||||
assert!(!pruner.should_prune(3, 1, ¤t, &completed));
|
||||
}
|
||||
|
||||
// --- Warmup steps ---
|
||||
|
||||
#[test]
|
||||
fn no_prune_during_warmup() {
|
||||
let pruner = MedianPruner::new(Direction::Minimize).n_warmup_steps(5);
|
||||
let completed = vec![trial_with_values(0, vec![(0, 1.0), (1, 1.0), (2, 1.0)])];
|
||||
// Step 2 < warmup 5 => never prune, even if value is terrible
|
||||
let current = vec![(0, 100.0), (1, 100.0), (2, 100.0)];
|
||||
assert!(!pruner.should_prune(1, 2, ¤t, &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prune_after_warmup() {
|
||||
let pruner = MedianPruner::new(Direction::Minimize).n_warmup_steps(2);
|
||||
let completed = vec![trial_with_values(0, vec![(0, 1.0), (1, 1.0), (2, 1.0)])];
|
||||
// Step 2 >= warmup 2 => pruning allowed; current 100.0 > median 1.0
|
||||
let current = vec![(0, 100.0), (1, 100.0), (2, 100.0)];
|
||||
assert!(pruner.should_prune(1, 2, ¤t, &completed));
|
||||
}
|
||||
|
||||
// --- n_min_trials ---
|
||||
|
||||
#[test]
|
||||
fn no_prune_when_fewer_than_n_min_trials() {
|
||||
let pruner = MedianPruner::new(Direction::Minimize).n_min_trials(3);
|
||||
// Only 2 completed trials — below threshold of 3
|
||||
let completed = vec![
|
||||
trial_with_values(0, vec![(0, 1.0)]),
|
||||
trial_with_values(1, vec![(0, 2.0)]),
|
||||
];
|
||||
let current = vec![(0, 100.0)];
|
||||
assert!(!pruner.should_prune(2, 0, ¤t, &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prune_when_at_least_n_min_trials() {
|
||||
let pruner = MedianPruner::new(Direction::Minimize).n_min_trials(3);
|
||||
// 3 completed trials with step 0: [1.0, 2.0, 3.0] => median 2.0
|
||||
let completed = vec![
|
||||
trial_with_values(0, vec![(0, 1.0)]),
|
||||
trial_with_values(1, vec![(0, 2.0)]),
|
||||
trial_with_values(2, vec![(0, 3.0)]),
|
||||
];
|
||||
// 5.0 > median 2.0 => prune
|
||||
let current = vec![(0, 5.0)];
|
||||
assert!(pruner.should_prune(3, 0, ¤t, &completed));
|
||||
}
|
||||
|
||||
// --- No completed trials with values at step ---
|
||||
|
||||
#[test]
|
||||
fn no_prune_when_no_completed_trials_at_step() {
|
||||
let pruner = MedianPruner::new(Direction::Minimize);
|
||||
// Completed trials only have values at step 0, not step 5
|
||||
let completed = vec![
|
||||
trial_with_values(0, vec![(0, 1.0)]),
|
||||
trial_with_values(1, vec![(0, 2.0)]),
|
||||
];
|
||||
let current = vec![(0, 0.5), (5, 100.0)];
|
||||
assert!(!pruner.should_prune(2, 5, ¤t, &completed));
|
||||
}
|
||||
|
||||
// --- Median calculation edge cases ---
|
||||
|
||||
#[test]
|
||||
fn correct_median_with_even_number_of_trials() {
|
||||
let pruner = MedianPruner::new(Direction::Minimize);
|
||||
// 4 trials at step 0: [1.0, 2.0, 3.0, 4.0] => median = 2.5
|
||||
let completed = vec![
|
||||
trial_with_values(0, vec![(0, 1.0)]),
|
||||
trial_with_values(1, vec![(0, 2.0)]),
|
||||
trial_with_values(2, vec![(0, 3.0)]),
|
||||
trial_with_values(3, vec![(0, 4.0)]),
|
||||
];
|
||||
// 2.6 > 2.5 => prune
|
||||
let current = vec![(0, 2.6)];
|
||||
assert!(pruner.should_prune(4, 0, ¤t, &completed));
|
||||
// 2.4 < 2.5 => don't prune
|
||||
let current = vec![(0, 2.4)];
|
||||
assert!(!pruner.should_prune(4, 0, ¤t, &completed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn correct_median_with_odd_number_of_trials() {
|
||||
let pruner = MedianPruner::new(Direction::Minimize);
|
||||
// 5 trials at step 0: [1.0, 2.0, 3.0, 4.0, 5.0] => median = 3.0
|
||||
let completed = vec![
|
||||
trial_with_values(0, vec![(0, 1.0)]),
|
||||
trial_with_values(1, vec![(0, 2.0)]),
|
||||
trial_with_values(2, vec![(0, 3.0)]),
|
||||
trial_with_values(3, vec![(0, 4.0)]),
|
||||
trial_with_values(4, vec![(0, 5.0)]),
|
||||
];
|
||||
// 3.5 > 3.0 => prune
|
||||
let current = vec![(0, 3.5)];
|
||||
assert!(pruner.should_prune(5, 0, ¤t, &completed));
|
||||
// 2.5 < 3.0 => don't prune
|
||||
let current = vec![(0, 2.5)];
|
||||
assert!(!pruner.should_prune(5, 0, ¤t, &completed));
|
||||
}
|
||||
|
||||
// --- Non-contiguous step numbers ---
|
||||
|
||||
#[test]
|
||||
fn works_with_non_contiguous_steps() {
|
||||
let pruner = MedianPruner::new(Direction::Minimize);
|
||||
// Steps are 0, 10, 100 — non-contiguous
|
||||
let completed = vec![
|
||||
trial_with_values(0, vec![(0, 1.0), (10, 2.0), (100, 3.0)]),
|
||||
trial_with_values(1, vec![(0, 1.5), (10, 2.5), (100, 4.0)]),
|
||||
trial_with_values(2, vec![(0, 2.0), (10, 3.0), (100, 5.0)]),
|
||||
];
|
||||
// At step 100: [3.0, 4.0, 5.0] => median = 4.0
|
||||
let current = vec![(0, 1.0), (10, 2.0), (100, 4.5)];
|
||||
assert!(pruner.should_prune(3, 100, ¤t, &completed));
|
||||
|
||||
let current = vec![(0, 1.0), (10, 2.0), (100, 3.5)];
|
||||
assert!(!pruner.should_prune(3, 100, ¤t, &completed));
|
||||
}
|
||||
|
||||
// --- No intermediate values for current trial ---
|
||||
|
||||
#[test]
|
||||
fn no_prune_when_no_intermediate_values() {
|
||||
let pruner = MedianPruner::new(Direction::Minimize);
|
||||
let completed = vec![trial_with_values(0, vec![(0, 1.0)])];
|
||||
assert!(!pruner.should_prune(1, 0, &[], &completed));
|
||||
}
|
||||
|
||||
// --- Pruned trials are excluded from median calculation ---
|
||||
|
||||
#[test]
|
||||
fn pruned_trials_excluded_from_median() {
|
||||
use optimizer::TrialState;
|
||||
|
||||
let pruner = MedianPruner::new(Direction::Minimize);
|
||||
|
||||
let mut pruned = trial_with_values(0, vec![(0, 0.1)]);
|
||||
pruned.state = TrialState::Pruned;
|
||||
|
||||
// Only the completed trial (value 5.0) counts. Pruned trial (0.1) is excluded.
|
||||
let completed = vec![pruned, trial_with_values(1, vec![(0, 5.0)])];
|
||||
|
||||
// 3.0 < 5.0 => don't prune (only 1 completed trial with median 5.0)
|
||||
let current = vec![(0, 3.0)];
|
||||
assert!(!pruner.should_prune(2, 0, ¤t, &completed));
|
||||
|
||||
// 6.0 > 5.0 => prune
|
||||
let current = vec![(0, 6.0)];
|
||||
assert!(pruner.should_prune(2, 0, ¤t, &completed));
|
||||
}
|
||||
@@ -1,393 +0,0 @@
|
||||
//! Integration tests for multi-objective optimization.
|
||||
|
||||
use optimizer::Direction;
|
||||
use optimizer::multi_objective::MultiObjectiveStudy;
|
||||
use optimizer::parameter::{CategoricalParam, FloatParam, Parameter};
|
||||
use optimizer::sampler::nsga2::Nsga2Sampler;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Pareto utility tests (via public MultiObjectiveStudy)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_basic_two_objective_random() {
|
||||
let study = MultiObjectiveStudy::new(vec![Direction::Minimize, Direction::Minimize]);
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
study
|
||||
.optimize(30, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(vec![xv, 1.0 - xv])
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let front = study.pareto_front();
|
||||
assert!(!front.is_empty(), "Pareto front should be non-empty");
|
||||
|
||||
// Verify no solution in the front dominates another
|
||||
for a in &front {
|
||||
for b in &front {
|
||||
if core::ptr::eq(a, b) {
|
||||
continue;
|
||||
}
|
||||
let a_dom_b = a.values[0] <= b.values[0]
|
||||
&& a.values[1] <= b.values[1]
|
||||
&& (a.values[0] < b.values[0] || a.values[1] < b.values[1]);
|
||||
assert!(
|
||||
!a_dom_b,
|
||||
"Front solution {:?} dominates {:?}",
|
||||
a.values, b.values
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dimension_mismatch_error() {
|
||||
let study = MultiObjectiveStudy::new(vec![Direction::Minimize, Direction::Minimize]);
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
let result = study.optimize(1, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
// Return wrong number of values
|
||||
Ok::<_, optimizer::Error>(vec![xv])
|
||||
});
|
||||
|
||||
assert!(result.is_err());
|
||||
let err = result.unwrap_err();
|
||||
assert!(
|
||||
matches!(
|
||||
err,
|
||||
optimizer::Error::ObjectiveDimensionMismatch {
|
||||
expected: 2,
|
||||
got: 1
|
||||
}
|
||||
),
|
||||
"Expected ObjectiveDimensionMismatch, got: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ask_tell() {
|
||||
let study = MultiObjectiveStudy::new(vec![Direction::Minimize, Direction::Maximize]);
|
||||
let x = FloatParam::new(0.0, 10.0);
|
||||
|
||||
for _ in 0..10 {
|
||||
let mut trial = study.ask();
|
||||
let xv = x.suggest(&mut trial).unwrap();
|
||||
study
|
||||
.tell(trial, Ok::<_, &str>(vec![xv, 10.0 - xv]))
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
assert_eq!(study.n_trials(), 10);
|
||||
let front = study.pareto_front();
|
||||
assert!(!front.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_ask_tell_dimension_mismatch() {
|
||||
let study = MultiObjectiveStudy::new(vec![Direction::Minimize, Direction::Minimize]);
|
||||
let trial = study.ask();
|
||||
let result = study.tell(trial, Ok::<_, &str>(vec![1.0, 2.0, 3.0]));
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_n_trials_counting() {
|
||||
let study = MultiObjectiveStudy::new(vec![Direction::Minimize, Direction::Minimize]);
|
||||
assert_eq!(study.n_trials(), 0);
|
||||
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
study
|
||||
.optimize(5, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(vec![xv, 1.0 - xv])
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(study.n_trials(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_three_objectives() {
|
||||
let study = MultiObjectiveStudy::new(vec![
|
||||
Direction::Minimize,
|
||||
Direction::Minimize,
|
||||
Direction::Maximize,
|
||||
]);
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
let y = FloatParam::new(0.0, 1.0);
|
||||
|
||||
study
|
||||
.optimize(30, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
let yv = y.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(vec![xv, yv, 1.0 - xv - yv])
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let front = study.pareto_front();
|
||||
assert!(!front.is_empty());
|
||||
assert_eq!(study.n_objectives(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_directions_accessor() {
|
||||
let dirs = vec![Direction::Minimize, Direction::Maximize];
|
||||
let study = MultiObjectiveStudy::new(dirs.clone());
|
||||
assert_eq!(study.directions(), &dirs);
|
||||
assert_eq!(study.n_objectives(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_trials_accessor() {
|
||||
let study = MultiObjectiveStudy::new(vec![Direction::Minimize, Direction::Minimize]);
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
study
|
||||
.optimize(3, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(vec![xv, 1.0 - xv])
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let trials = study.trials();
|
||||
assert_eq!(trials.len(), 3);
|
||||
for t in &trials {
|
||||
assert_eq!(t.values.len(), 2);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// NSGA-II sampler tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[test]
|
||||
fn test_nsga2_zdt1() {
|
||||
// ZDT1 benchmark: minimize both objectives
|
||||
let n_vars = 5;
|
||||
let params: Vec<FloatParam> = (0..n_vars).map(|_| FloatParam::new(0.0, 1.0)).collect();
|
||||
|
||||
let sampler = Nsga2Sampler::builder().population_size(20).seed(42).build();
|
||||
let study =
|
||||
MultiObjectiveStudy::with_sampler(vec![Direction::Minimize, Direction::Minimize], sampler);
|
||||
|
||||
study
|
||||
.optimize(200, |trial| {
|
||||
let xs: Vec<f64> = params
|
||||
.iter()
|
||||
.map(|p| p.suggest(trial))
|
||||
.collect::<Result<_, _>>()?;
|
||||
|
||||
let f1 = xs[0];
|
||||
let g = 1.0 + 9.0 * xs[1..].iter().sum::<f64>() / (n_vars - 1) as f64;
|
||||
let f2 = g * (1.0 - (f1 / g).sqrt());
|
||||
Ok::<_, optimizer::Error>(vec![f1, f2])
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let front = study.pareto_front();
|
||||
assert!(!front.is_empty(), "Pareto front should be non-empty");
|
||||
|
||||
// Verify no dominated solutions in the front
|
||||
for a in &front {
|
||||
for b in &front {
|
||||
if core::ptr::eq(a, b) {
|
||||
continue;
|
||||
}
|
||||
let a_dom_b = a.values[0] <= b.values[0]
|
||||
&& a.values[1] <= b.values[1]
|
||||
&& (a.values[0] < b.values[0] || a.values[1] < b.values[1]);
|
||||
assert!(
|
||||
!a_dom_b,
|
||||
"Front solution {:?} dominates {:?}",
|
||||
a.values, b.values
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_nsga2_with_seed_reproducible() {
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
let y = FloatParam::new(0.0, 1.0);
|
||||
|
||||
let run = |seed: u64| -> Vec<Vec<f64>> {
|
||||
let sampler = Nsga2Sampler::with_seed(seed);
|
||||
let study = MultiObjectiveStudy::with_sampler(
|
||||
vec![Direction::Minimize, Direction::Minimize],
|
||||
sampler,
|
||||
);
|
||||
study
|
||||
.optimize(30, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
let yv = y.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(vec![xv, yv])
|
||||
})
|
||||
.unwrap();
|
||||
study.trials().iter().map(|t| t.values.clone()).collect()
|
||||
};
|
||||
|
||||
let r1 = run(123);
|
||||
let r2 = run(123);
|
||||
assert_eq!(r1, r2, "Same seed should produce same results");
|
||||
|
||||
let r3 = run(456);
|
||||
assert_ne!(r1, r3, "Different seeds should produce different results");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_nsga2_builder() {
|
||||
let sampler = Nsga2Sampler::builder()
|
||||
.population_size(10)
|
||||
.crossover_prob(0.8)
|
||||
.crossover_eta(15.0)
|
||||
.mutation_eta(25.0)
|
||||
.seed(42)
|
||||
.build();
|
||||
|
||||
let study =
|
||||
MultiObjectiveStudy::with_sampler(vec![Direction::Minimize, Direction::Minimize], sampler);
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
study
|
||||
.optimize(30, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(vec![xv, 1.0 - xv])
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(study.n_trials(), 30);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_nsga2_categorical_params() {
|
||||
let sampler = Nsga2Sampler::with_seed(42);
|
||||
let study =
|
||||
MultiObjectiveStudy::with_sampler(vec![Direction::Minimize, Direction::Minimize], sampler);
|
||||
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
let cat = CategoricalParam::new(vec!["a", "b", "c"]);
|
||||
|
||||
study
|
||||
.optimize(30, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
let cv = cat.suggest(trial)?;
|
||||
let bonus = match cv {
|
||||
"a" => 0.0,
|
||||
"b" => 0.5,
|
||||
_ => 1.0,
|
||||
};
|
||||
Ok::<_, optimizer::Error>(vec![xv + bonus, 1.0 - xv])
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(study.n_trials(), 30);
|
||||
let front = study.pareto_front();
|
||||
assert!(!front.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_nsga2_constraints() {
|
||||
let sampler = Nsga2Sampler::with_seed(42);
|
||||
let study =
|
||||
MultiObjectiveStudy::with_sampler(vec![Direction::Minimize, Direction::Minimize], sampler);
|
||||
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
study
|
||||
.optimize(50, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
// Constraint: x >= 0.3 (i.e. 0.3 - x <= 0)
|
||||
trial.set_constraints(vec![0.3 - xv]);
|
||||
Ok::<_, optimizer::Error>(vec![xv, 1.0 - xv])
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let front = study.pareto_front();
|
||||
assert!(!front.is_empty());
|
||||
|
||||
// Check that feasible solutions exist on the front
|
||||
let feasible_count = front.iter().filter(|t| t.is_feasible()).count();
|
||||
assert!(
|
||||
feasible_count > 0,
|
||||
"Should have feasible solutions on front"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multi_objective_trial_get() {
|
||||
let study = MultiObjectiveStudy::new(vec![Direction::Minimize, Direction::Minimize]);
|
||||
let x = FloatParam::new(0.0, 10.0).name("x");
|
||||
|
||||
study
|
||||
.optimize(5, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(vec![xv, 10.0 - xv])
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let front = study.pareto_front();
|
||||
for t in &front {
|
||||
let xv: f64 = t.get(&x).unwrap();
|
||||
assert!((0.0..=10.0).contains(&xv));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multi_objective_trial_is_feasible() {
|
||||
let study = MultiObjectiveStudy::new(vec![Direction::Minimize, Direction::Minimize]);
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
study
|
||||
.optimize(10, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
trial.set_constraints(vec![0.5 - xv]); // feasible if x >= 0.5
|
||||
Ok::<_, optimizer::Error>(vec![xv, 1.0 - xv])
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let trials = study.trials();
|
||||
for t in &trials {
|
||||
let xv = t.values[0];
|
||||
if xv >= 0.5 {
|
||||
assert!(t.is_feasible());
|
||||
} else {
|
||||
assert!(!t.is_feasible());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multi_objective_trial_user_attrs() {
|
||||
let study = MultiObjectiveStudy::new(vec![Direction::Minimize, Direction::Minimize]);
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
study
|
||||
.optimize(3, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
trial.set_user_attr("iteration", 42_i64);
|
||||
Ok::<_, optimizer::Error>(vec![xv, 1.0 - xv])
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let trials = study.trials();
|
||||
for t in &trials {
|
||||
assert!(t.user_attr("iteration").is_some());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tell_with_failure() {
|
||||
let study = MultiObjectiveStudy::new(vec![Direction::Minimize, Direction::Minimize]);
|
||||
|
||||
let trial = study.ask();
|
||||
study
|
||||
.tell(trial, Err::<Vec<f64>, _>("evaluation failed"))
|
||||
.unwrap();
|
||||
|
||||
// Failed trial not counted
|
||||
assert_eq!(study.n_trials(), 0);
|
||||
}
|
||||
@@ -1,424 +0,0 @@
|
||||
//! Integration tests for the Multivariate TPE sampler.
|
||||
//!
|
||||
//! These tests compare the performance of `MultivariateTpeSampler` against
|
||||
//! the standard `TpeSampler` on problems with and without parameter correlations.
|
||||
|
||||
#![allow(
|
||||
clippy::cast_sign_loss,
|
||||
clippy::cast_precision_loss,
|
||||
clippy::cast_possible_truncation
|
||||
)]
|
||||
|
||||
use optimizer::parameter::{CategoricalParam, FloatParam, IntParam, Parameter};
|
||||
use optimizer::sampler::tpe::{MultivariateTpeSampler, TpeSampler};
|
||||
use optimizer::{Direction, Error, Study};
|
||||
|
||||
// =============================================================================
|
||||
// Rosenbrock function: f(x,y) = (a-x)^2 + b*(y-x^2)^2
|
||||
// This is a classic benchmark with strong parameter correlation.
|
||||
// Optimal: (x, y) = (a, a^2) with f(x, y) = 0
|
||||
// Standard parameters: a = 1, b = 100
|
||||
// The "banana-shaped" valley makes this hard for independent samplers.
|
||||
// =============================================================================
|
||||
|
||||
/// Computes the Rosenbrock function value.
|
||||
///
|
||||
/// f(x, y) = (a - x)^2 + b * (y - x^2)^2
|
||||
///
|
||||
/// With standard parameters a = 1, b = 100:
|
||||
/// - Optimal point: (1, 1)
|
||||
/// - Optimal value: 0
|
||||
fn rosenbrock(x: f64, y: f64) -> f64 {
|
||||
let a = 1.0;
|
||||
let b = 100.0;
|
||||
(a - x).powi(2) + b * (y - x * x).powi(2)
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Test: Multivariate TPE on Rosenbrock function (correlated parameters)
|
||||
// =============================================================================
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_tpe_rosenbrock_finds_good_solution() {
|
||||
// Multivariate TPE should find a good solution on Rosenbrock
|
||||
// because it can model the correlation between x and y
|
||||
let sampler = MultivariateTpeSampler::builder()
|
||||
.seed(42)
|
||||
.n_startup_trials(10)
|
||||
.n_ei_candidates(24)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x_param = FloatParam::new(-2.0, 2.0);
|
||||
let y_param = FloatParam::new(-2.0, 4.0);
|
||||
|
||||
study
|
||||
.optimize(100, |trial| {
|
||||
let x = x_param.suggest(trial)?;
|
||||
let y = y_param.suggest(trial)?;
|
||||
Ok::<_, Error>(rosenbrock(x, y))
|
||||
})
|
||||
.expect("optimization should succeed");
|
||||
|
||||
let best = study.best_trial().expect("should have at least one trial");
|
||||
|
||||
// Multivariate TPE should find a reasonably good solution
|
||||
// The global minimum is 0, but getting close is challenging
|
||||
assert!(
|
||||
best.value < 10.0,
|
||||
"Multivariate TPE should find good Rosenbrock solution: best value {} should be < 10.0",
|
||||
best.value
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_independent_tpe_rosenbrock() {
|
||||
// Independent TPE (standard TpeSampler) on Rosenbrock
|
||||
// This establishes a baseline for comparison
|
||||
let sampler = TpeSampler::builder()
|
||||
.seed(42)
|
||||
.n_startup_trials(10)
|
||||
.n_ei_candidates(24)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x_param = FloatParam::new(-2.0, 2.0);
|
||||
let y_param = FloatParam::new(-2.0, 4.0);
|
||||
|
||||
study
|
||||
.optimize(100, |trial| {
|
||||
let x = x_param.suggest(trial)?;
|
||||
let y = y_param.suggest(trial)?;
|
||||
Ok::<_, Error>(rosenbrock(x, y))
|
||||
})
|
||||
.expect("optimization should succeed");
|
||||
|
||||
let best = study.best_trial().expect("should have at least one trial");
|
||||
|
||||
// Independent TPE should still find a decent solution,
|
||||
// but may not be as good as multivariate TPE on this correlated problem
|
||||
assert!(
|
||||
best.value < 50.0,
|
||||
"Independent TPE should find reasonable Rosenbrock solution: best value {} should be < 50.0",
|
||||
best.value
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_tpe_outperforms_on_correlated_problem() {
|
||||
// Run multiple seeds and compare average performance
|
||||
// Multivariate TPE should generally find better solutions on Rosenbrock
|
||||
let n_runs = 5;
|
||||
let n_trials = 80;
|
||||
|
||||
let mut multivariate_best_values = Vec::new();
|
||||
let mut independent_best_values = Vec::new();
|
||||
|
||||
for seed in 0..n_runs {
|
||||
// Multivariate TPE
|
||||
let multivariate_sampler = MultivariateTpeSampler::builder()
|
||||
.seed(seed as u64)
|
||||
.n_startup_trials(10)
|
||||
.n_ei_candidates(24)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, multivariate_sampler);
|
||||
|
||||
let x_param = FloatParam::new(-2.0, 2.0);
|
||||
let y_param = FloatParam::new(-2.0, 4.0);
|
||||
|
||||
study
|
||||
.optimize(n_trials, |trial| {
|
||||
let x = x_param.suggest(trial)?;
|
||||
let y = y_param.suggest(trial)?;
|
||||
Ok::<_, Error>(rosenbrock(x, y))
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
multivariate_best_values.push(study.best_trial().unwrap().value);
|
||||
|
||||
// Independent TPE
|
||||
let independent_sampler = TpeSampler::builder()
|
||||
.seed(seed as u64)
|
||||
.n_startup_trials(10)
|
||||
.n_ei_candidates(24)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, independent_sampler);
|
||||
|
||||
let x_param = FloatParam::new(-2.0, 2.0);
|
||||
let y_param = FloatParam::new(-2.0, 4.0);
|
||||
|
||||
study
|
||||
.optimize(n_trials, |trial| {
|
||||
let x = x_param.suggest(trial)?;
|
||||
let y = y_param.suggest(trial)?;
|
||||
Ok::<_, Error>(rosenbrock(x, y))
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
independent_best_values.push(study.best_trial().unwrap().value);
|
||||
}
|
||||
|
||||
let multivariate_mean: f64 = multivariate_best_values.iter().sum::<f64>() / n_runs as f64;
|
||||
let independent_mean: f64 = independent_best_values.iter().sum::<f64>() / n_runs as f64;
|
||||
|
||||
// Log results for debugging (these won't show in normal test runs,
|
||||
// but are useful when running with --nocapture)
|
||||
eprintln!("Multivariate TPE mean best: {multivariate_mean:.4}");
|
||||
eprintln!("Independent TPE mean best: {independent_mean:.4}");
|
||||
eprintln!("Multivariate best values: {multivariate_best_values:?}");
|
||||
eprintln!("Independent best values: {independent_best_values:?}");
|
||||
|
||||
// Both methods should find reasonable solutions
|
||||
assert!(
|
||||
multivariate_mean < 20.0,
|
||||
"Multivariate TPE mean {multivariate_mean:.4} should be < 20.0"
|
||||
);
|
||||
assert!(
|
||||
independent_mean < 100.0,
|
||||
"Independent TPE mean {independent_mean:.4} should be < 100.0"
|
||||
);
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Independent parameter problem: f(x,y) = x^2 + y^2
|
||||
// No correlation between parameters - both methods should work equally well.
|
||||
// =============================================================================
|
||||
|
||||
/// Simple sphere function with independent parameters.
|
||||
///
|
||||
/// f(x, y) = x^2 + y^2
|
||||
///
|
||||
/// Optimal point: (0, 0)
|
||||
/// Optimal value: 0
|
||||
fn sphere(x: f64, y: f64) -> f64 {
|
||||
x * x + y * y
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_tpe_independent_problem() {
|
||||
// On an independent problem, multivariate TPE should still work well
|
||||
let sampler = MultivariateTpeSampler::builder()
|
||||
.seed(42)
|
||||
.n_startup_trials(10)
|
||||
.n_ei_candidates(24)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x_param = FloatParam::new(-5.0, 5.0);
|
||||
let y_param = FloatParam::new(-5.0, 5.0);
|
||||
|
||||
study
|
||||
.optimize(50, |trial| {
|
||||
let x = x_param.suggest(trial)?;
|
||||
let y = y_param.suggest(trial)?;
|
||||
Ok::<_, Error>(sphere(x, y))
|
||||
})
|
||||
.expect("optimization should succeed");
|
||||
|
||||
let best = study.best_trial().expect("should have at least one trial");
|
||||
|
||||
assert!(
|
||||
best.value < 5.0,
|
||||
"Multivariate TPE should find good solution on sphere: best value {} should be < 5.0",
|
||||
best.value
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_independent_tpe_independent_problem() {
|
||||
// Baseline: independent TPE on sphere function
|
||||
let sampler = TpeSampler::builder()
|
||||
.seed(42)
|
||||
.n_startup_trials(10)
|
||||
.n_ei_candidates(24)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x_param = FloatParam::new(-5.0, 5.0);
|
||||
let y_param = FloatParam::new(-5.0, 5.0);
|
||||
|
||||
study
|
||||
.optimize(50, |trial| {
|
||||
let x = x_param.suggest(trial)?;
|
||||
let y = y_param.suggest(trial)?;
|
||||
Ok::<_, Error>(sphere(x, y))
|
||||
})
|
||||
.expect("optimization should succeed");
|
||||
|
||||
let best = study.best_trial().expect("should have at least one trial");
|
||||
|
||||
assert!(
|
||||
best.value < 5.0,
|
||||
"Independent TPE should find good solution on sphere: best value {} should be < 5.0",
|
||||
best.value
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_both_samplers_work_on_independent_problem() {
|
||||
// Run both samplers on the independent sphere function
|
||||
// and verify they both achieve similar performance
|
||||
let n_runs = 5;
|
||||
let n_trials = 50;
|
||||
|
||||
let mut multivariate_results = Vec::new();
|
||||
let mut independent_results = Vec::new();
|
||||
|
||||
for seed in 0..n_runs {
|
||||
// Multivariate TPE
|
||||
let sampler = MultivariateTpeSampler::builder()
|
||||
.seed(seed as u64)
|
||||
.n_startup_trials(10)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x_param = FloatParam::new(-5.0, 5.0);
|
||||
let y_param = FloatParam::new(-5.0, 5.0);
|
||||
|
||||
study
|
||||
.optimize(n_trials, |trial| {
|
||||
let x = x_param.suggest(trial)?;
|
||||
let y = y_param.suggest(trial)?;
|
||||
Ok::<_, Error>(sphere(x, y))
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
multivariate_results.push(study.best_trial().unwrap().value);
|
||||
|
||||
// Independent TPE
|
||||
let sampler = TpeSampler::builder()
|
||||
.seed(seed as u64)
|
||||
.n_startup_trials(10)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x_param = FloatParam::new(-5.0, 5.0);
|
||||
let y_param = FloatParam::new(-5.0, 5.0);
|
||||
|
||||
study
|
||||
.optimize(n_trials, |trial| {
|
||||
let x = x_param.suggest(trial)?;
|
||||
let y = y_param.suggest(trial)?;
|
||||
Ok::<_, Error>(sphere(x, y))
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
independent_results.push(study.best_trial().unwrap().value);
|
||||
}
|
||||
|
||||
let multivariate_mean: f64 = multivariate_results.iter().sum::<f64>() / n_runs as f64;
|
||||
let independent_mean: f64 = independent_results.iter().sum::<f64>() / n_runs as f64;
|
||||
|
||||
eprintln!("Sphere function results:");
|
||||
eprintln!(" Multivariate TPE mean: {multivariate_mean:.4}");
|
||||
eprintln!(" Independent TPE mean: {independent_mean:.4}");
|
||||
|
||||
// Both should find good solutions on this simple problem
|
||||
assert!(
|
||||
multivariate_mean < 5.0,
|
||||
"Multivariate TPE mean {multivariate_mean:.4} should be < 5.0 on sphere"
|
||||
);
|
||||
assert!(
|
||||
independent_mean < 5.0,
|
||||
"Independent TPE mean {independent_mean:.4} should be < 5.0 on sphere"
|
||||
);
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Test: Multivariate TPE with group decomposition
|
||||
// =============================================================================
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_tpe_with_group_decomposition() {
|
||||
// Test that group decomposition works correctly
|
||||
let sampler = MultivariateTpeSampler::builder()
|
||||
.seed(42)
|
||||
.n_startup_trials(10)
|
||||
.group(true) // Enable group decomposition
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x_param = FloatParam::new(-5.0, 5.0);
|
||||
let y_param = FloatParam::new(-5.0, 5.0);
|
||||
|
||||
study
|
||||
.optimize(50, |trial| {
|
||||
let x = x_param.suggest(trial)?;
|
||||
let y = y_param.suggest(trial)?;
|
||||
Ok::<_, Error>(sphere(x, y))
|
||||
})
|
||||
.expect("optimization should succeed");
|
||||
|
||||
let best = study.best_trial().expect("should have at least one trial");
|
||||
|
||||
assert!(
|
||||
best.value < 10.0,
|
||||
"Multivariate TPE with groups should find good solution: best value {} should be < 10.0",
|
||||
best.value
|
||||
);
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Test: Multivariate TPE with mixed parameter types
|
||||
// =============================================================================
|
||||
|
||||
#[test]
|
||||
fn test_multivariate_tpe_mixed_parameter_types() {
|
||||
// Test with float, int, and categorical parameters
|
||||
let sampler = MultivariateTpeSampler::builder()
|
||||
.seed(42)
|
||||
.n_startup_trials(10)
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x_param = FloatParam::new(-5.0, 5.0);
|
||||
let n_param = IntParam::new(1, 10);
|
||||
let mode_param = CategoricalParam::new(vec!["a", "b", "c"]);
|
||||
|
||||
study
|
||||
.optimize(50, |trial| {
|
||||
let x = x_param.suggest(trial)?;
|
||||
let n = n_param.suggest(trial)?;
|
||||
let mode = mode_param.suggest(trial)?;
|
||||
|
||||
// Objective depends on all parameters
|
||||
let mode_factor = match mode {
|
||||
"a" => 1.0,
|
||||
"b" => 0.5,
|
||||
"c" => 2.0,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
|
||||
Ok::<_, Error>(x * x + (n as f64 - 5.0).powi(2) * mode_factor)
|
||||
})
|
||||
.expect("optimization should succeed");
|
||||
|
||||
let best = study.best_trial().expect("should have at least one trial");
|
||||
|
||||
// Should find a reasonable solution
|
||||
assert!(
|
||||
best.value < 25.0,
|
||||
"Multivariate TPE should handle mixed types: best value {} should be < 25.0",
|
||||
best.value
|
||||
);
|
||||
}
|
||||
@@ -1,240 +0,0 @@
|
||||
use optimizer::parameter::{
|
||||
BoolParam, Categorical, CategoricalParam, EnumParam, FloatParam, IntParam, Parameter,
|
||||
};
|
||||
use optimizer::{Direction, Study, Trial};
|
||||
|
||||
#[test]
|
||||
fn suggest_float_param_via_trial() {
|
||||
let param = FloatParam::new(0.0, 1.0);
|
||||
let mut trial = Trial::new(0);
|
||||
let x = trial.suggest_param(¶m).unwrap();
|
||||
assert!((0.0..=1.0).contains(&x));
|
||||
|
||||
// Cached
|
||||
let x2 = trial.suggest_param(¶m).unwrap();
|
||||
assert_eq!(x, x2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suggest_float_log_param_via_trial() {
|
||||
let param = FloatParam::new(1e-5, 1e-1).log_scale();
|
||||
let mut trial = Trial::new(0);
|
||||
let lr = trial.suggest_param(¶m).unwrap();
|
||||
assert!((1e-5..=1e-1).contains(&lr));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suggest_float_step_param_via_trial() {
|
||||
let param = FloatParam::new(0.0, 1.0).step(0.25);
|
||||
let mut trial = Trial::new(0);
|
||||
let x = trial.suggest_param(¶m).unwrap();
|
||||
assert!((0.0..=1.0).contains(&x));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suggest_int_param_via_trial() {
|
||||
let param = IntParam::new(1, 10);
|
||||
let mut trial = Trial::new(0);
|
||||
let n = trial.suggest_param(¶m).unwrap();
|
||||
assert!((1..=10).contains(&n));
|
||||
|
||||
// Cached
|
||||
let n2 = trial.suggest_param(¶m).unwrap();
|
||||
assert_eq!(n, n2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suggest_int_log_param_via_trial() {
|
||||
let param = IntParam::new(1, 1024).log_scale();
|
||||
let mut trial = Trial::new(0);
|
||||
let batch = trial.suggest_param(¶m).unwrap();
|
||||
assert!((1..=1024).contains(&batch));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suggest_int_step_param_via_trial() {
|
||||
let param = IntParam::new(32, 512).step(32);
|
||||
let mut trial = Trial::new(0);
|
||||
let units = trial.suggest_param(¶m).unwrap();
|
||||
assert!((32..=512).contains(&units));
|
||||
assert_eq!((units - 32) % 32, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suggest_categorical_param_via_trial() {
|
||||
let choices = vec!["sgd", "adam", "rmsprop"];
|
||||
let param = CategoricalParam::new(choices.clone());
|
||||
let mut trial = Trial::new(0);
|
||||
let opt = trial.suggest_param(¶m).unwrap();
|
||||
assert!(choices.contains(&opt));
|
||||
|
||||
// Cached
|
||||
let opt2 = trial.suggest_param(¶m).unwrap();
|
||||
assert_eq!(opt, opt2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suggest_bool_param_via_trial() {
|
||||
let param = BoolParam::new();
|
||||
let mut trial = Trial::new(0);
|
||||
let val = trial.suggest_param(¶m).unwrap();
|
||||
let _ = val;
|
||||
|
||||
// Cached
|
||||
let val2 = trial.suggest_param(¶m).unwrap();
|
||||
assert_eq!(val, val2);
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
enum Activation {
|
||||
Relu,
|
||||
Sigmoid,
|
||||
Tanh,
|
||||
}
|
||||
|
||||
impl Categorical for Activation {
|
||||
const N_CHOICES: usize = 3;
|
||||
|
||||
fn from_index(index: usize) -> Self {
|
||||
match index {
|
||||
0 => Activation::Relu,
|
||||
1 => Activation::Sigmoid,
|
||||
2 => Activation::Tanh,
|
||||
_ => panic!("invalid index"),
|
||||
}
|
||||
}
|
||||
|
||||
fn to_index(&self) -> usize {
|
||||
match self {
|
||||
Activation::Relu => 0,
|
||||
Activation::Sigmoid => 1,
|
||||
Activation::Tanh => 2,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suggest_enum_param_via_trial() {
|
||||
let param = EnumParam::<Activation>::new();
|
||||
let mut trial = Trial::new(0);
|
||||
let act = trial.suggest_param(¶m).unwrap();
|
||||
assert!([Activation::Relu, Activation::Sigmoid, Activation::Tanh].contains(&act));
|
||||
|
||||
// Cached
|
||||
let act2 = trial.suggest_param(¶m).unwrap();
|
||||
assert_eq!(act, act2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parameter_conflict_detection() {
|
||||
let float_param = FloatParam::new(0.0, 1.0);
|
||||
let int_param = IntParam::new(0, 10);
|
||||
let mut trial = Trial::new(0);
|
||||
let _ = trial.suggest_param(&float_param).unwrap();
|
||||
|
||||
// Different param type with different id - no conflict
|
||||
let result = trial.suggest_param(&int_param);
|
||||
assert!(result.is_ok());
|
||||
|
||||
// Different bounds for same param type but different id - no conflict
|
||||
let float_param2 = FloatParam::new(0.0, 2.0);
|
||||
let result = trial.suggest_param(&float_param2);
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validation_prevents_suggest() {
|
||||
let mut trial = Trial::new(0);
|
||||
|
||||
assert!(trial.suggest_param(&FloatParam::new(1.0, 0.0)).is_err());
|
||||
assert!(
|
||||
trial
|
||||
.suggest_param(&FloatParam::new(-1.0, 1.0).log_scale())
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
trial
|
||||
.suggest_param(&FloatParam::new(0.0, 1.0).step(-0.1))
|
||||
.is_err()
|
||||
);
|
||||
assert!(trial.suggest_param(&IntParam::new(10, 1)).is_err());
|
||||
assert!(
|
||||
trial
|
||||
.suggest_param(&IntParam::new(0, 10).log_scale())
|
||||
.is_err()
|
||||
);
|
||||
assert!(trial.suggest_param(&IntParam::new(0, 10).step(-1)).is_err());
|
||||
assert!(
|
||||
trial
|
||||
.suggest_param(&CategoricalParam::<&str>::new(vec![]))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parameter_api_with_study() {
|
||||
let x_param = FloatParam::new(-5.0, 5.0);
|
||||
let n_param = IntParam::new(1, 10);
|
||||
let dropout_param = BoolParam::new();
|
||||
let opt_param = CategoricalParam::new(vec!["sgd", "adam"]);
|
||||
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
study
|
||||
.optimize(5, |trial| {
|
||||
let x = x_param.suggest(trial)?;
|
||||
let n = n_param.suggest(trial)?;
|
||||
let dropout = dropout_param.suggest(trial)?;
|
||||
let opt = opt_param.suggest(trial)?;
|
||||
let _ = (n, dropout, opt);
|
||||
Ok::<_, optimizer::Error>(x * x)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let best = study.best_trial().unwrap();
|
||||
assert!(best.value >= 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parameter_suggest_method() {
|
||||
let param = FloatParam::new(0.0, 1.0);
|
||||
let mut trial = Trial::new(0);
|
||||
let x = param.suggest(&mut trial).unwrap();
|
||||
assert!((0.0..=1.0).contains(&x));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn existing_suggest_methods_still_work() {
|
||||
let mut trial = Trial::new(0);
|
||||
|
||||
let x_param = FloatParam::new(0.0, 1.0);
|
||||
let x = x_param.suggest(&mut trial).unwrap();
|
||||
assert!((0.0..=1.0).contains(&x));
|
||||
|
||||
let lr_param = FloatParam::new(1e-5, 1e-1).log_scale();
|
||||
let lr = lr_param.suggest(&mut trial).unwrap();
|
||||
assert!((1e-5..=1e-1).contains(&lr));
|
||||
|
||||
let step_param = FloatParam::new(0.0, 1.0).step(0.25);
|
||||
let step = step_param.suggest(&mut trial).unwrap();
|
||||
assert!((0.0..=1.0).contains(&step));
|
||||
|
||||
let n_param = IntParam::new(1, 10);
|
||||
let n = n_param.suggest(&mut trial).unwrap();
|
||||
assert!((1..=10).contains(&n));
|
||||
|
||||
let batch_param = IntParam::new(1, 1024).log_scale();
|
||||
let batch = batch_param.suggest(&mut trial).unwrap();
|
||||
assert!((1..=1024).contains(&batch));
|
||||
|
||||
let units_param = IntParam::new(32, 512).step(32);
|
||||
let units = units_param.suggest(&mut trial).unwrap();
|
||||
assert!((32..=512).contains(&units));
|
||||
|
||||
let opt_param = CategoricalParam::new(vec!["sgd", "adam", "rmsprop"]);
|
||||
let opt = opt_param.suggest(&mut trial).unwrap();
|
||||
assert!(["sgd", "adam", "rmsprop"].contains(&opt));
|
||||
|
||||
let flag_param = BoolParam::new();
|
||||
let flag = flag_param.suggest(&mut trial).unwrap();
|
||||
let _ = flag;
|
||||
}
|
||||
@@ -1,295 +0,0 @@
|
||||
#![cfg(feature = "serde")]
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use optimizer::parameter::{FloatParam, IntParam, Parameter};
|
||||
use optimizer::sampler::CompletedTrial;
|
||||
use optimizer::{Direction, ParamValue, Study, StudySnapshot, TrialState};
|
||||
|
||||
#[test]
|
||||
fn round_trip_save_load() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let x = FloatParam::new(-10.0, 10.0).name("x");
|
||||
let n = IntParam::new(1, 100).name("n");
|
||||
|
||||
study
|
||||
.optimize(5, |trial| {
|
||||
let x_val = x.suggest(trial)?;
|
||||
let n_val = n.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(x_val * x_val + n_val as f64)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let dir = tempdir();
|
||||
let path = dir.join("study.json");
|
||||
|
||||
study.save(&path).unwrap();
|
||||
let loaded: Study<f64> = Study::load(&path).unwrap();
|
||||
|
||||
assert_eq!(loaded.direction(), study.direction());
|
||||
assert_eq!(loaded.n_trials(), study.n_trials());
|
||||
|
||||
let orig_trials = study.trials();
|
||||
let loaded_trials = loaded.trials();
|
||||
|
||||
for (orig, loaded) in orig_trials.iter().zip(loaded_trials.iter()) {
|
||||
assert_eq!(orig.id, loaded.id);
|
||||
assert!((orig.value - loaded.value).abs() < 1e-10);
|
||||
assert_eq!(orig.state, loaded.state);
|
||||
assert_eq!(orig.params.len(), loaded.params.len());
|
||||
assert_eq!(orig.distributions, loaded.distributions);
|
||||
assert_eq!(orig.param_labels, loaded.param_labels);
|
||||
}
|
||||
|
||||
// Clean up
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_output_is_human_readable() {
|
||||
let study: Study<f64> = Study::new(Direction::Maximize);
|
||||
let x = FloatParam::new(0.0, 1.0).name("x");
|
||||
|
||||
study
|
||||
.optimize(2, |trial| {
|
||||
let v = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(v)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let dir = tempdir();
|
||||
let path = dir.join("study.json");
|
||||
study.save(&path).unwrap();
|
||||
|
||||
let contents = std::fs::read_to_string(&path).unwrap();
|
||||
|
||||
// Verify it's pretty-printed JSON with recognizable fields
|
||||
assert!(contents.contains("\"version\""));
|
||||
assert!(contents.contains("\"direction\""));
|
||||
assert!(contents.contains("\"trials\""));
|
||||
assert!(contents.contains("\"next_trial_id\""));
|
||||
assert!(contents.contains("\"Maximize\""));
|
||||
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trip_empty_study() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
|
||||
let dir = tempdir();
|
||||
let path = dir.join("empty.json");
|
||||
|
||||
study.save(&path).unwrap();
|
||||
let loaded: Study<f64> = Study::load(&path).unwrap();
|
||||
|
||||
assert_eq!(loaded.direction(), Direction::Minimize);
|
||||
assert_eq!(loaded.n_trials(), 0);
|
||||
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_version_field_is_present() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
|
||||
let dir = tempdir();
|
||||
let path = dir.join("version.json");
|
||||
study.save(&path).unwrap();
|
||||
|
||||
let contents = std::fs::read_to_string(&path).unwrap();
|
||||
let snapshot: StudySnapshot<f64> = serde_json::from_str(&contents).unwrap();
|
||||
|
||||
assert_eq!(snapshot.version, 1);
|
||||
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_trial_serde_round_trip() {
|
||||
let trial = CompletedTrial::new(42, HashMap::new(), HashMap::new(), HashMap::new(), 2.78);
|
||||
|
||||
let json = serde_json::to_string(&trial).unwrap();
|
||||
let deserialized: CompletedTrial<f64> = serde_json::from_str(&json).unwrap();
|
||||
|
||||
assert_eq!(deserialized.id, 42);
|
||||
assert_eq!(deserialized.value, 2.78);
|
||||
assert_eq!(deserialized.state, TrialState::Complete);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn param_value_serde_round_trip() {
|
||||
let values = vec![
|
||||
ParamValue::Float(1.23),
|
||||
ParamValue::Int(42),
|
||||
ParamValue::Categorical(2),
|
||||
];
|
||||
|
||||
for val in &values {
|
||||
let json = serde_json::to_string(val).unwrap();
|
||||
let deserialized: ParamValue = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(&deserialized, val);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn direction_serde_round_trip() {
|
||||
let min_json = serde_json::to_string(&Direction::Minimize).unwrap();
|
||||
let max_json = serde_json::to_string(&Direction::Maximize).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Direction>(&min_json).unwrap(),
|
||||
Direction::Minimize
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_str::<Direction>(&max_json).unwrap(),
|
||||
Direction::Maximize
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trip_preserves_trial_id_counter() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
study
|
||||
.optimize(10, |trial| {
|
||||
let v = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(v)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let dir = tempdir();
|
||||
let path = dir.join("counter.json");
|
||||
study.save(&path).unwrap();
|
||||
|
||||
let loaded: Study<f64> = Study::load(&path).unwrap();
|
||||
|
||||
// Creating a new trial should use an ID >= 10
|
||||
let trial = loaded.create_trial();
|
||||
assert!(trial.id() >= 10);
|
||||
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_file_created_at_interval() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let x = FloatParam::new(-10.0, 10.0).name("x");
|
||||
|
||||
let dir = tempdir();
|
||||
let checkpoint = dir.join("checkpoint.json");
|
||||
|
||||
study
|
||||
.optimize_with_checkpoint(10, 3, &checkpoint, |trial| {
|
||||
let v = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(v * v)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// Checkpoint should exist (written at trials 3, 6, 9)
|
||||
assert!(checkpoint.exists(), "checkpoint file was not created");
|
||||
|
||||
// Load it and verify it's valid
|
||||
let loaded: Study<f64> = Study::load(&checkpoint).unwrap();
|
||||
// Last checkpoint was at trial 9, so it should have 9 trials
|
||||
assert_eq!(loaded.n_trials(), 9);
|
||||
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_overwrites_previous() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
let dir = tempdir();
|
||||
let checkpoint = dir.join("checkpoint.json");
|
||||
|
||||
study
|
||||
.optimize_with_checkpoint(6, 3, &checkpoint, |trial| {
|
||||
let v = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(v)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// The checkpoint at trial 6 should overwrite the one from trial 3
|
||||
let loaded: Study<f64> = Study::load(&checkpoint).unwrap();
|
||||
assert_eq!(loaded.n_trials(), 6);
|
||||
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resume_from_checkpoint_continues_trial_ids() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let x = FloatParam::new(-5.0, 5.0).name("x");
|
||||
|
||||
let dir = tempdir();
|
||||
let checkpoint = dir.join("resume.json");
|
||||
|
||||
// Run 10 trials with checkpointing
|
||||
study
|
||||
.optimize_with_checkpoint(10, 5, &checkpoint, |trial| {
|
||||
let v = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(v * v)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// Load and continue
|
||||
let loaded: Study<f64> = Study::load(&checkpoint).unwrap();
|
||||
assert_eq!(loaded.n_trials(), 10);
|
||||
|
||||
let remaining = 15 - loaded.n_trials();
|
||||
loaded
|
||||
.optimize(remaining, |trial| {
|
||||
let v = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(v * v)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(loaded.n_trials(), 15);
|
||||
|
||||
// Verify no duplicate trial IDs
|
||||
let trials = loaded.trials();
|
||||
let mut ids: Vec<u64> = trials.iter().map(|t| t.id).collect();
|
||||
ids.sort_unstable();
|
||||
ids.dedup();
|
||||
assert_eq!(ids.len(), 15, "duplicate trial IDs found");
|
||||
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn atomic_write_no_temp_file_left_behind() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
let dir = tempdir();
|
||||
let checkpoint = dir.join("atomic.json");
|
||||
|
||||
study
|
||||
.optimize_with_checkpoint(3, 3, &checkpoint, |trial| {
|
||||
let v = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(v)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// The temp file should have been renamed, not left behind
|
||||
let tmp_path = dir.join(".atomic.json.tmp");
|
||||
assert!(!tmp_path.exists(), "temp file was not cleaned up");
|
||||
assert!(checkpoint.exists(), "checkpoint file was not created");
|
||||
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
/// Helper to create a unique temporary directory.
|
||||
fn tempdir() -> std::path::PathBuf {
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
static COUNTER: AtomicU64 = AtomicU64::new(0);
|
||||
let id = COUNTER.fetch_add(1, Ordering::Relaxed);
|
||||
let dir =
|
||||
std::env::temp_dir().join(format!("optimizer_serde_test_{}_{id}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
dir
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
#[path = "../benches/test_functions.rs"]
|
||||
mod test_functions;
|
||||
|
||||
use test_functions::*;
|
||||
|
||||
const TOL: f64 = 1e-10;
|
||||
|
||||
#[test]
|
||||
fn sphere_at_optimum() {
|
||||
assert!(sphere(&[0.0, 0.0]).abs() < TOL);
|
||||
assert!(sphere(&[0.0; 10]).abs() < TOL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rosenbrock_at_optimum() {
|
||||
assert!(rosenbrock(&[1.0, 1.0]).abs() < TOL);
|
||||
assert!(rosenbrock(&[1.0; 5]).abs() < TOL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rastrigin_at_optimum() {
|
||||
assert!(rastrigin(&[0.0, 0.0]).abs() < TOL);
|
||||
assert!(rastrigin(&[0.0; 10]).abs() < TOL);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ackley_at_optimum() {
|
||||
assert!(ackley(&[0.0, 0.0]).abs() < 1e-8);
|
||||
assert!(ackley(&[0.0; 10]).abs() < 1e-8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn branin_at_optimum() {
|
||||
let target = 0.397_887_357_729_738_1;
|
||||
let val = branin(&[std::f64::consts::PI, 2.275]);
|
||||
assert!((val - target).abs() < 1e-3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hartmann6_at_optimum() {
|
||||
let target = -3.3224;
|
||||
let x_opt = [0.20169, 0.150011, 0.476874, 0.275332, 0.311652, 0.6573];
|
||||
let val = hartmann6(&x_opt);
|
||||
assert!((val - target).abs() < 0.01);
|
||||
}
|
||||
@@ -1,64 +0,0 @@
|
||||
use optimizer::pruner::{Pruner, ThresholdPruner};
|
||||
|
||||
#[test]
|
||||
fn prune_when_value_exceeds_upper_threshold() {
|
||||
let pruner = ThresholdPruner::new().upper(10.0);
|
||||
let values = vec![(0, 5.0), (1, 8.0), (2, 11.0)];
|
||||
assert!(pruner.should_prune(0, 2, &values, &[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prune_when_value_falls_below_lower_threshold() {
|
||||
let pruner = ThresholdPruner::new().lower(0.0);
|
||||
let values = vec![(0, 5.0), (1, 2.0), (2, -1.0)];
|
||||
assert!(pruner.should_prune(0, 2, &values, &[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_prune_when_value_within_bounds() {
|
||||
let pruner = ThresholdPruner::new().upper(10.0).lower(0.0);
|
||||
let values = vec![(0, 3.0), (1, 5.0), (2, 7.0)];
|
||||
assert!(!pruner.should_prune(0, 2, &values, &[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_prune_when_no_intermediate_values() {
|
||||
let pruner = ThresholdPruner::new().upper(10.0).lower(0.0);
|
||||
assert!(!pruner.should_prune(0, 0, &[], &[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn works_with_only_upper_set() {
|
||||
let pruner = ThresholdPruner::new().upper(5.0);
|
||||
let below = vec![(0, 3.0)];
|
||||
let above = vec![(0, 6.0)];
|
||||
assert!(!pruner.should_prune(0, 0, &below, &[]));
|
||||
assert!(pruner.should_prune(0, 0, &above, &[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn works_with_only_lower_set() {
|
||||
let pruner = ThresholdPruner::new().lower(2.0);
|
||||
let above = vec![(0, 5.0)];
|
||||
let below = vec![(0, 1.0)];
|
||||
assert!(!pruner.should_prune(0, 0, &above, &[]));
|
||||
assert!(pruner.should_prune(0, 0, &below, &[]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn works_with_both_thresholds_set() {
|
||||
let pruner = ThresholdPruner::new().upper(10.0).lower(0.0);
|
||||
|
||||
// Within bounds
|
||||
assert!(!pruner.should_prune(0, 0, &[(0, 5.0)], &[]));
|
||||
|
||||
// Exceeds upper
|
||||
assert!(pruner.should_prune(0, 0, &[(0, 15.0)], &[]));
|
||||
|
||||
// Below lower
|
||||
assert!(pruner.should_prune(0, 0, &[(0, -3.0)], &[]));
|
||||
|
||||
// At exact boundary (not pruned — strictly greater/less)
|
||||
assert!(!pruner.should_prune(0, 0, &[(0, 10.0)], &[]));
|
||||
assert!(!pruner.should_prune(0, 0, &[(0, 0.0)], &[]));
|
||||
}
|
||||
@@ -1,148 +0,0 @@
|
||||
use optimizer::parameter::{FloatParam, Parameter};
|
||||
use optimizer::{AttrValue, Direction, Study};
|
||||
|
||||
#[test]
|
||||
fn set_and_get_float_attr() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
study
|
||||
.optimize(1, |trial| {
|
||||
let _ = x.suggest(trial)?;
|
||||
trial.set_user_attr("score", 42.5);
|
||||
assert_eq!(trial.user_attr("score"), Some(&AttrValue::Float(42.5)));
|
||||
Ok::<_, optimizer::Error>(1.0)
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_and_get_int_attr() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
study
|
||||
.optimize(1, |trial| {
|
||||
let _ = x.suggest(trial)?;
|
||||
trial.set_user_attr("epoch", 42_i64);
|
||||
assert_eq!(trial.user_attr("epoch"), Some(&AttrValue::Int(42)));
|
||||
Ok::<_, optimizer::Error>(1.0)
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_and_get_string_attr() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
study
|
||||
.optimize(1, |trial| {
|
||||
let _ = x.suggest(trial)?;
|
||||
trial.set_user_attr("model", "resnet50");
|
||||
assert_eq!(
|
||||
trial.user_attr("model"),
|
||||
Some(&AttrValue::String("resnet50".to_owned()))
|
||||
);
|
||||
Ok::<_, optimizer::Error>(1.0)
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_and_get_bool_attr() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
study
|
||||
.optimize(1, |trial| {
|
||||
let _ = x.suggest(trial)?;
|
||||
trial.set_user_attr("converged", true);
|
||||
assert_eq!(trial.user_attr("converged"), Some(&AttrValue::Bool(true)));
|
||||
Ok::<_, optimizer::Error>(1.0)
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attrs_propagate_to_completed_trial() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
study
|
||||
.optimize(1, |trial| {
|
||||
let _ = x.suggest(trial)?;
|
||||
trial.set_user_attr("time_secs", 1.5);
|
||||
trial.set_user_attr("tag", "baseline");
|
||||
Ok::<_, optimizer::Error>(1.0)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let best = study.best_trial().unwrap();
|
||||
assert_eq!(best.user_attr("time_secs"), Some(&AttrValue::Float(1.5)));
|
||||
assert_eq!(
|
||||
best.user_attr("tag"),
|
||||
Some(&AttrValue::String("baseline".to_owned()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overwrite_attr_replaces_value() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
study
|
||||
.optimize(1, |trial| {
|
||||
let _ = x.suggest(trial)?;
|
||||
trial.set_user_attr("key", "old");
|
||||
trial.set_user_attr("key", "new");
|
||||
assert_eq!(
|
||||
trial.user_attr("key"),
|
||||
Some(&AttrValue::String("new".to_owned()))
|
||||
);
|
||||
Ok::<_, optimizer::Error>(1.0)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let best = study.best_trial().unwrap();
|
||||
assert_eq!(
|
||||
best.user_attr("key"),
|
||||
Some(&AttrValue::String("new".to_owned()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_attr_returns_none() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
study
|
||||
.optimize(1, |trial| {
|
||||
let _ = x.suggest(trial)?;
|
||||
assert_eq!(trial.user_attr("nonexistent"), None);
|
||||
Ok::<_, optimizer::Error>(1.0)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let best = study.best_trial().unwrap();
|
||||
assert_eq!(best.user_attr("nonexistent"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_attrs_map_returns_all() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
let x = FloatParam::new(0.0, 1.0);
|
||||
|
||||
study
|
||||
.optimize(1, |trial| {
|
||||
let _ = x.suggest(trial)?;
|
||||
trial.set_user_attr("a", 1.0);
|
||||
trial.set_user_attr("b", true);
|
||||
assert_eq!(trial.user_attrs().len(), 2);
|
||||
Ok::<_, optimizer::Error>(1.0)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let best = study.best_trial().unwrap();
|
||||
assert_eq!(best.user_attrs().len(), 2);
|
||||
}
|
||||
@@ -1,182 +0,0 @@
|
||||
#![cfg(feature = "visualization")]
|
||||
|
||||
use optimizer::parameter::{FloatParam, IntParam, Parameter};
|
||||
use optimizer::sampler::random::RandomSampler;
|
||||
use optimizer::{Direction, Study, generate_html_report};
|
||||
|
||||
#[test]
|
||||
fn html_report_creates_file() {
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(42));
|
||||
let x = FloatParam::new(0.0, 10.0).name("x");
|
||||
let y = IntParam::new(1, 5).name("y");
|
||||
|
||||
study
|
||||
.optimize(10, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
let yv = y.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(xv + yv as f64)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let path = std::env::temp_dir().join("test_report_creates_file.html");
|
||||
generate_html_report(&study, &path).unwrap();
|
||||
|
||||
let content = std::fs::read_to_string(&path).unwrap();
|
||||
assert!(content.contains("<!DOCTYPE html>"));
|
||||
assert!(content.contains("plotly"));
|
||||
std::fs::remove_file(&path).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn html_report_contains_all_chart_sections() {
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(42));
|
||||
let x = FloatParam::new(0.0, 10.0).name("x");
|
||||
let y = FloatParam::new(-5.0, 5.0).name("y");
|
||||
|
||||
study
|
||||
.optimize(20, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
let yv = y.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(xv * xv + yv * yv)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let path = std::env::temp_dir().join("test_report_all_charts.html");
|
||||
generate_html_report(&study, &path).unwrap();
|
||||
|
||||
let content = std::fs::read_to_string(&path).unwrap();
|
||||
|
||||
// Should contain all chart divs.
|
||||
assert!(content.contains("id=\"history\""));
|
||||
assert!(content.contains("id=\"slices\""));
|
||||
assert!(content.contains("id=\"parcoords\""));
|
||||
assert!(content.contains("id=\"importance\""));
|
||||
assert!(content.contains("id=\"timeline\""));
|
||||
|
||||
// Should contain chart titles.
|
||||
assert!(content.contains("Optimization History"));
|
||||
assert!(content.contains("Slice Plots"));
|
||||
assert!(content.contains("Parallel Coordinates"));
|
||||
assert!(content.contains("Parameter Importance"));
|
||||
assert!(content.contains("Trial Timeline"));
|
||||
|
||||
// Should show direction and trial count.
|
||||
assert!(content.contains("Minimize"));
|
||||
assert!(content.contains("20 trials"));
|
||||
|
||||
std::fs::remove_file(&path).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn html_report_empty_study() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
|
||||
let path = std::env::temp_dir().join("test_report_empty.html");
|
||||
generate_html_report(&study, &path).unwrap();
|
||||
|
||||
let content = std::fs::read_to_string(&path).unwrap();
|
||||
assert!(content.contains("<!DOCTYPE html>"));
|
||||
assert!(content.contains("0 trials"));
|
||||
|
||||
std::fs::remove_file(&path).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn html_report_single_param_no_parcoords() {
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(42));
|
||||
let x = FloatParam::new(0.0, 10.0).name("x");
|
||||
|
||||
study
|
||||
.optimize(5, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(xv * xv)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let path = std::env::temp_dir().join("test_report_single_param.html");
|
||||
generate_html_report(&study, &path).unwrap();
|
||||
|
||||
let content = std::fs::read_to_string(&path).unwrap();
|
||||
|
||||
// Should have slice plot but not parallel coordinates (needs >= 2 params).
|
||||
assert!(content.contains("id=\"slices\""));
|
||||
assert!(!content.contains("id=\"parcoords\""));
|
||||
|
||||
std::fs::remove_file(&path).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn html_report_maximize_direction() {
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Maximize, RandomSampler::with_seed(42));
|
||||
let x = FloatParam::new(0.0, 10.0).name("x");
|
||||
|
||||
study
|
||||
.optimize(5, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(xv)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let path = std::env::temp_dir().join("test_report_maximize.html");
|
||||
generate_html_report(&study, &path).unwrap();
|
||||
|
||||
let content = std::fs::read_to_string(&path).unwrap();
|
||||
assert!(content.contains("Maximize"));
|
||||
|
||||
std::fs::remove_file(&path).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_html_convenience_method() {
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(42));
|
||||
let x = FloatParam::new(0.0, 10.0).name("x");
|
||||
|
||||
study
|
||||
.optimize(5, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(xv * xv)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let path = std::env::temp_dir().join("test_export_html.html");
|
||||
study.export_html(&path).unwrap();
|
||||
|
||||
let content = std::fs::read_to_string(&path).unwrap();
|
||||
assert!(content.contains("<!DOCTYPE html>"));
|
||||
assert!(content.contains("id=\"history\""));
|
||||
|
||||
std::fs::remove_file(&path).ok();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn html_report_with_intermediate_values() {
|
||||
use optimizer::pruner::MedianPruner;
|
||||
|
||||
let mut study: Study<f64> =
|
||||
Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(42));
|
||||
study.set_pruner(MedianPruner::new(Direction::Minimize));
|
||||
let x = FloatParam::new(0.0, 10.0).name("x");
|
||||
|
||||
study
|
||||
.optimize(10, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
for step in 0..5 {
|
||||
let val = xv * xv + step as f64;
|
||||
trial.report(step, val);
|
||||
if trial.should_prune() {
|
||||
return Err(optimizer::TrialPruned.into());
|
||||
}
|
||||
}
|
||||
Ok::<_, optimizer::Error>(xv * xv)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let path = std::env::temp_dir().join("test_report_intermediate.html");
|
||||
generate_html_report(&study, &path).unwrap();
|
||||
|
||||
let content = std::fs::read_to_string(&path).unwrap();
|
||||
assert!(content.contains("id=\"intermediate\""));
|
||||
assert!(content.contains("Intermediate Values"));
|
||||
|
||||
std::fs::remove_file(&path).ok();
|
||||
}
|
||||
@@ -1,3 +1,2 @@
|
||||
[default.extend-words]
|
||||
Tpe = "Tpe"
|
||||
consts = "consts"
|
||||
|
||||
Reference in New Issue
Block a user