refactor(docs): Documentation Overhaul
This commit is contained in:
Manuel Raimann
@@ -0,0 +1,277 @@
|
||||
//! Advanced Features Example
|
||||
//!
|
||||
//! This example demonstrates four advanced capabilities of the optimizer crate:
|
||||
//!
|
||||
//! 1. **Async parallel optimization** — evaluate multiple trials concurrently
|
||||
//! 2. **Journal storage** — persist trials to disk and resume studies later
|
||||
//! 3. **Ask-and-tell interface** — decouple sampling from evaluation
|
||||
//! 4. **Multi-objective optimization** — optimize competing objectives simultaneously
|
||||
//!
|
||||
//! Run with: `cargo run --example advanced_features --features "async,journal"`
|
||||
|
||||
use std::time::Instant;
|
||||
|
||||
use optimizer::multi_objective::MultiObjectiveStudy;
|
||||
use optimizer::prelude::*;
|
||||
|
||||
// ============================================================================
|
||||
// Section 1: Async Parallel Optimization
|
||||
// ============================================================================
|
||||
|
||||
/// Runs multiple trials concurrently using tokio, reducing wall-clock time
|
||||
/// when the objective function involves I/O or other async work.
|
||||
async fn async_parallel_optimization() -> optimizer::Result<()> {
|
||||
println!("=== Section 1: Async Parallel Optimization ===\n");
|
||||
|
||||
let sampler = TpeSampler::builder()
|
||||
.n_startup_trials(5)
|
||||
.seed(42)
|
||||
.build()
|
||||
.expect("Failed to build TPE sampler");
|
||||
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
let x = FloatParam::new(-5.0, 5.0).name("x");
|
||||
let y = FloatParam::new(-5.0, 5.0).name("y");
|
||||
|
||||
let n_trials = 30;
|
||||
let concurrency = 4;
|
||||
|
||||
println!("Running {n_trials} trials with {concurrency} concurrent workers...");
|
||||
let start = Instant::now();
|
||||
|
||||
// optimize_parallel spawns up to `concurrency` trials at once.
|
||||
// The closure must take ownership of Trial and return (Trial, value).
|
||||
study
|
||||
.optimize_parallel(n_trials, concurrency, {
|
||||
let x = x.clone();
|
||||
let y = y.clone();
|
||||
move |mut trial| {
|
||||
let x = x.clone();
|
||||
let y = y.clone();
|
||||
async move {
|
||||
let xv = x.suggest(&mut trial)?;
|
||||
let yv = y.suggest(&mut trial)?;
|
||||
|
||||
// Simulate async I/O (e.g., calling an external service)
|
||||
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
|
||||
|
||||
// Sphere function: minimum at origin
|
||||
let value = xv * xv + yv * yv;
|
||||
Ok::<_, optimizer::Error>((trial, value))
|
||||
}
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
let best = study.best_trial()?;
|
||||
|
||||
println!(
|
||||
"Completed in {elapsed:.2?} (vs ~{:.0?} sequential)",
|
||||
std::time::Duration::from_millis(10 * n_trials as u64)
|
||||
);
|
||||
println!(
|
||||
"Best: f({:.3}, {:.3}) = {:.6}\n",
|
||||
best.get(&x).unwrap(),
|
||||
best.get(&y).unwrap(),
|
||||
best.value
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Section 2: Journal Storage
|
||||
// ============================================================================
|
||||
|
||||
/// Persists trials to a JSONL file so that a study can be resumed later.
|
||||
/// Useful for long-running experiments or crash recovery.
|
||||
fn journal_storage_demo() -> optimizer::Result<()> {
|
||||
println!("=== Section 2: Journal Storage ===\n");
|
||||
|
||||
let path = std::env::temp_dir().join("optimizer_advanced_example.jsonl");
|
||||
|
||||
// Clean up from any previous run
|
||||
let _ = std::fs::remove_file(&path);
|
||||
|
||||
let x = FloatParam::new(-5.0, 5.0).name("x");
|
||||
|
||||
// --- First run: optimize 20 trials and persist to disk ---
|
||||
{
|
||||
let storage = JournalStorage::<f64>::new(&path);
|
||||
let study: Study<f64> = Study::builder()
|
||||
.minimize()
|
||||
.sampler(TpeSampler::new())
|
||||
.storage(storage)
|
||||
.build();
|
||||
|
||||
study.optimize(20, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(xv * xv)
|
||||
})?;
|
||||
|
||||
println!(
|
||||
"First run: {} trials saved to {}",
|
||||
study.n_trials(),
|
||||
path.display()
|
||||
);
|
||||
}
|
||||
|
||||
// --- Second run: resume from the journal file ---
|
||||
{
|
||||
// JournalStorage::open loads existing trials from disk
|
||||
let storage = JournalStorage::<f64>::open(&path)?;
|
||||
let study: Study<f64> = Study::builder()
|
||||
.minimize()
|
||||
.sampler(TpeSampler::new())
|
||||
.storage(storage)
|
||||
.build();
|
||||
|
||||
// The sampler sees the prior 20 trials, so it starts informed
|
||||
let before = study.n_trials();
|
||||
study.optimize(10, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
Ok::<_, optimizer::Error>(xv * xv)
|
||||
})?;
|
||||
|
||||
let best = study.best_trial()?;
|
||||
println!(
|
||||
"Resumed: {} → {} trials, best f({:.4}) = {:.6}",
|
||||
before,
|
||||
study.n_trials(),
|
||||
best.get(&x).unwrap(),
|
||||
best.value
|
||||
);
|
||||
}
|
||||
|
||||
// Clean up the temporary file
|
||||
let _ = std::fs::remove_file(&path);
|
||||
|
||||
println!();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Section 3: Ask-and-Tell Interface
|
||||
// ============================================================================
|
||||
|
||||
/// Decouples trial creation from evaluation. Useful when:
|
||||
/// - Evaluations happen outside the optimizer (e.g., in a separate process)
|
||||
/// - You want to batch evaluations before reporting results
|
||||
/// - You need custom scheduling logic
|
||||
fn ask_and_tell_demo() -> optimizer::Result<()> {
|
||||
println!("=== Section 3: Ask-and-Tell Interface ===\n");
|
||||
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
|
||||
let x = FloatParam::new(-5.0, 5.0).name("x");
|
||||
let y = FloatParam::new(-5.0, 5.0).name("y");
|
||||
|
||||
// Ask for a batch of trials, evaluate externally, then tell results
|
||||
for batch in 0..3 {
|
||||
let batch_size = 5;
|
||||
let mut trials = Vec::with_capacity(batch_size);
|
||||
|
||||
// ask() creates trials with sampled parameters
|
||||
for _ in 0..batch_size {
|
||||
let mut trial = study.ask();
|
||||
let xv = x.suggest(&mut trial)?;
|
||||
let yv = y.suggest(&mut trial)?;
|
||||
|
||||
// Store values alongside the trial for later evaluation
|
||||
trials.push((trial, xv, yv));
|
||||
}
|
||||
|
||||
// Evaluate the batch (could be sent to workers, GPUs, etc.)
|
||||
for (trial, xv, yv) in trials {
|
||||
let value = xv * xv + yv * yv;
|
||||
// tell() reports the result back to the study
|
||||
study.tell(trial, Ok::<_, &str>(value));
|
||||
}
|
||||
|
||||
println!(
|
||||
"Batch {}: evaluated {} trials (total: {})",
|
||||
batch + 1,
|
||||
batch_size,
|
||||
study.n_trials()
|
||||
);
|
||||
}
|
||||
|
||||
let best = study.best_trial()?;
|
||||
println!(
|
||||
"Best: f({:.3}, {:.3}) = {:.6}\n",
|
||||
best.get(&x).unwrap(),
|
||||
best.get(&y).unwrap(),
|
||||
best.value
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Section 4: Multi-Objective Optimization
|
||||
// ============================================================================
|
||||
|
||||
/// Optimizes two competing objectives simultaneously.
|
||||
/// Returns the Pareto front — the set of solutions where no objective can
|
||||
/// be improved without worsening the other.
|
||||
fn multi_objective_demo() -> optimizer::Result<()> {
|
||||
println!("=== Section 4: Multi-Objective Optimization ===\n");
|
||||
|
||||
// Two objectives, both minimized
|
||||
let study = MultiObjectiveStudy::new(vec![Direction::Minimize, Direction::Minimize]);
|
||||
|
||||
let x = FloatParam::new(0.0, 1.0).name("x");
|
||||
|
||||
// Classic bi-objective problem: f1(x) = x², f2(x) = (x - 1)²
|
||||
// The Pareto front is the curve where improving f1 worsens f2 and vice versa.
|
||||
study.optimize(50, |trial| {
|
||||
let xv = x.suggest(trial)?;
|
||||
let f1 = xv * xv;
|
||||
let f2 = (xv - 1.0) * (xv - 1.0);
|
||||
Ok::<_, optimizer::Error>(vec![f1, f2])
|
||||
})?;
|
||||
|
||||
let front = study.pareto_front();
|
||||
println!(
|
||||
"Ran {} trials, Pareto front has {} solutions:",
|
||||
study.n_trials(),
|
||||
front.len()
|
||||
);
|
||||
|
||||
// Show a few Pareto-optimal trade-offs
|
||||
let mut sorted_front = front.clone();
|
||||
sorted_front.sort_by(|a, b| a.values[0].partial_cmp(&b.values[0]).unwrap());
|
||||
|
||||
for (i, trial) in sorted_front.iter().take(5).enumerate() {
|
||||
println!(
|
||||
" {}: x={:.3}, f1={:.4}, f2={:.4}",
|
||||
i + 1,
|
||||
trial.get(&x).unwrap(),
|
||||
trial.values[0],
|
||||
trial.values[1]
|
||||
);
|
||||
}
|
||||
if sorted_front.len() > 5 {
|
||||
println!(" ... and {} more", sorted_front.len() - 5);
|
||||
}
|
||||
|
||||
println!();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Main
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> optimizer::Result<()> {
|
||||
async_parallel_optimization().await?;
|
||||
journal_storage_demo()?;
|
||||
ask_and_tell_demo()?;
|
||||
multi_objective_demo()?;
|
||||
|
||||
println!("All sections completed successfully!");
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,368 +0,0 @@
|
||||
//! Async API Parameter Optimization Example
|
||||
//!
|
||||
//! This example shows how to use async/parallel optimization to tune
|
||||
//! configuration parameters for a web service. Each evaluation simulates
|
||||
//! an async operation (like deploying and load-testing a service).
|
||||
//!
|
||||
//! # Key Concepts Demonstrated
|
||||
//!
|
||||
//! - Async optimization with `optimize_parallel`
|
||||
//! - Running multiple trials concurrently for faster optimization
|
||||
//! - Boolean and categorical parameter types
|
||||
//! - Measuring speedup from parallelism
|
||||
//!
|
||||
//! # When to Use Async Optimization
|
||||
//!
|
||||
//! Use async/parallel optimization when your objective function involves:
|
||||
//! - Network requests (API calls, database queries)
|
||||
//! - File I/O operations
|
||||
//! - External service calls
|
||||
//! - Any operation where you're waiting for I/O rather than computing
|
||||
//!
|
||||
//! With parallelism, you can evaluate multiple configurations simultaneously,
|
||||
//! significantly reducing total optimization time.
|
||||
//!
|
||||
//! Run with: `cargo run --example async_api_optimization --features async`
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use optimizer::prelude::*;
|
||||
|
||||
// ============================================================================
|
||||
// Configuration: Service parameters we want to tune
|
||||
// ============================================================================
|
||||
|
||||
/// Configuration for a web service.
|
||||
///
|
||||
/// In a real application, these parameters would control:
|
||||
/// - Memory allocation (cache sizes)
|
||||
/// - Connection management (pool sizes, timeouts)
|
||||
/// - Request handling (batching, compression)
|
||||
/// - Protocol options (HTTP version, load balancing)
|
||||
struct ServiceConfig {
|
||||
cache_size_mb: i64,
|
||||
connection_pool_size: i64,
|
||||
request_timeout_ms: i64,
|
||||
retry_count: i64,
|
||||
batch_size: i64,
|
||||
compression_level: i64,
|
||||
use_http2: bool,
|
||||
load_balancing: String,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Objective Function: Evaluate a service configuration
|
||||
// ============================================================================
|
||||
|
||||
/// Simulates deploying and load-testing a service configuration.
|
||||
///
|
||||
/// In a real scenario, this function might:
|
||||
/// 1. Deploy the configuration to a staging environment
|
||||
/// 2. Run load tests against the service
|
||||
/// 3. Collect metrics (latency, throughput, error rate)
|
||||
/// 4. Return a composite score
|
||||
///
|
||||
/// The async sleep simulates the I/O time of these operations.
|
||||
/// This is where parallel execution helps - while one trial is waiting
|
||||
/// for I/O, other trials can run.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn evaluate_service(config: &ServiceConfig) -> f64 {
|
||||
// Simulate async I/O (deployment, load testing, metric collection)
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
|
||||
// Calculate a score based on how close we are to optimal values
|
||||
// Lower score = better configuration
|
||||
let mut score = 0.0;
|
||||
|
||||
// Cache size: too small = cache misses, too large = wasted memory
|
||||
// Optimal around 512MB
|
||||
let cache_optimal = 512.0;
|
||||
score += ((config.cache_size_mb as f64 - cache_optimal) / 256.0).powi(2);
|
||||
|
||||
// Connection pool: too small = contention, too large = resource waste
|
||||
// Optimal around 100
|
||||
let pool_optimal = 100.0;
|
||||
score += ((config.connection_pool_size as f64 - pool_optimal) / 50.0).powi(2);
|
||||
|
||||
// Timeout: too short = false failures, too long = slow recovery
|
||||
// Optimal around 5000ms
|
||||
let timeout_optimal = 5000.0;
|
||||
score += ((config.request_timeout_ms as f64 - timeout_optimal) / 2000.0).powi(2);
|
||||
|
||||
// Retries: too few = fragile, too many = amplifies failures
|
||||
// Optimal around 3
|
||||
let retry_optimal = 3.0;
|
||||
score += ((config.retry_count as f64 - retry_optimal) / 2.0).powi(2);
|
||||
|
||||
// Batch size: trade-off between latency and throughput
|
||||
// Optimal around 64
|
||||
let batch_optimal = 64.0;
|
||||
score += ((config.batch_size as f64 - batch_optimal) / 32.0).powi(2);
|
||||
|
||||
// Compression level: trade-off between CPU and bandwidth
|
||||
// Optimal around 6
|
||||
let compression_optimal = 6.0;
|
||||
score += ((config.compression_level as f64 - compression_optimal) / 3.0).powi(2);
|
||||
|
||||
// HTTP/2 is generally better for our use case
|
||||
if !config.use_http2 {
|
||||
score += 0.5;
|
||||
}
|
||||
|
||||
// Load balancing strategy affects performance
|
||||
score += match config.load_balancing.as_str() {
|
||||
"round_robin" => 0.0, // Best for our use case
|
||||
"least_connections" => 0.1, // Good alternative
|
||||
"ip_hash" => 0.2, // OK for session affinity
|
||||
"random" => 0.3, // Not ideal
|
||||
_ => 1.0,
|
||||
};
|
||||
|
||||
// Add noise to simulate real-world variability
|
||||
let noise = (config.cache_size_mb as f64 * 0.1).sin() * 0.05;
|
||||
|
||||
score + noise
|
||||
}
|
||||
|
||||
/// The async objective function for each trial.
|
||||
///
|
||||
/// For async optimization, the objective function must:
|
||||
/// 1. Take ownership of the Trial (not a mutable reference)
|
||||
/// 2. Return a Future
|
||||
/// 3. Return both the Trial and the result value as a tuple
|
||||
///
|
||||
/// This ownership pattern allows the trial to be used across await points.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn objective(
|
||||
mut trial: Trial,
|
||||
cache_size_mb_param: &IntParam,
|
||||
connection_pool_size_param: &IntParam,
|
||||
request_timeout_ms_param: &IntParam,
|
||||
retry_count_param: &IntParam,
|
||||
batch_size_param: &IntParam,
|
||||
compression_level_param: &IntParam,
|
||||
use_http2_param: &BoolParam,
|
||||
load_balancing_param: &CategoricalParam<&str>,
|
||||
) -> optimizer::Result<(Trial, f64)> {
|
||||
// Sample configuration parameters using parameter definitions
|
||||
let cache_size_mb = cache_size_mb_param.suggest(&mut trial)?;
|
||||
let connection_pool_size = connection_pool_size_param.suggest(&mut trial)?;
|
||||
let request_timeout_ms = request_timeout_ms_param.suggest(&mut trial)?;
|
||||
let retry_count = retry_count_param.suggest(&mut trial)?;
|
||||
let batch_size = batch_size_param.suggest(&mut trial)?;
|
||||
let compression_level = compression_level_param.suggest(&mut trial)?;
|
||||
let use_http2 = use_http2_param.suggest(&mut trial)?;
|
||||
let load_balancing = load_balancing_param.suggest(&mut trial)?;
|
||||
|
||||
// Build configuration
|
||||
let config = ServiceConfig {
|
||||
cache_size_mb,
|
||||
connection_pool_size,
|
||||
request_timeout_ms,
|
||||
retry_count,
|
||||
batch_size,
|
||||
compression_level,
|
||||
use_http2,
|
||||
load_balancing: load_balancing.to_string(),
|
||||
};
|
||||
|
||||
// Evaluate (this is the async part)
|
||||
let score = evaluate_service(&config).await;
|
||||
|
||||
// Return both the trial and the score
|
||||
Ok((trial, score))
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Helper Functions
|
||||
// ============================================================================
|
||||
|
||||
/// Prints the results of the optimization.
|
||||
fn print_results(study: &Study<f64>, elapsed: Duration, n_trials: usize) {
|
||||
println!("\n{}", "=".repeat(60));
|
||||
println!("\nOptimization completed!");
|
||||
println!("Total trials: {}", study.n_trials());
|
||||
println!("Time elapsed: {elapsed:.2?}");
|
||||
|
||||
// Calculate speedup from parallelism
|
||||
// Each trial takes ~50ms, so sequential would take n_trials * 50ms
|
||||
let sequential_time = n_trials as f64 * 0.050;
|
||||
let actual_time = elapsed.as_secs_f64();
|
||||
println!(
|
||||
"Effective parallelism: {:.1}x speedup",
|
||||
sequential_time / actual_time
|
||||
);
|
||||
}
|
||||
|
||||
/// Prints the best configuration found.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn print_best_config(
|
||||
study: &Study<f64>,
|
||||
cache_size_mb_param: &IntParam,
|
||||
connection_pool_size_param: &IntParam,
|
||||
request_timeout_ms_param: &IntParam,
|
||||
retry_count_param: &IntParam,
|
||||
batch_size_param: &IntParam,
|
||||
compression_level_param: &IntParam,
|
||||
use_http2_param: &BoolParam,
|
||||
load_balancing_param: &CategoricalParam<&str>,
|
||||
) -> optimizer::Result<()> {
|
||||
let best = study.best_trial()?;
|
||||
|
||||
println!("\nBest configuration found:");
|
||||
println!(" Score: {:.6}", best.value);
|
||||
println!("\n Parameters:");
|
||||
println!(
|
||||
" cache_size_mb: {}",
|
||||
best.get(cache_size_mb_param).unwrap()
|
||||
);
|
||||
println!(
|
||||
" connection_pool_size: {}",
|
||||
best.get(connection_pool_size_param).unwrap()
|
||||
);
|
||||
println!(
|
||||
" request_timeout_ms: {}",
|
||||
best.get(request_timeout_ms_param).unwrap()
|
||||
);
|
||||
println!(" retry_count: {}", best.get(retry_count_param).unwrap());
|
||||
println!(" batch_size: {}", best.get(batch_size_param).unwrap());
|
||||
println!(
|
||||
" compression_level: {}",
|
||||
best.get(compression_level_param).unwrap()
|
||||
);
|
||||
println!(" use_http2: {}", best.get(use_http2_param).unwrap());
|
||||
println!(
|
||||
" load_balancing: {}",
|
||||
best.get(load_balancing_param).unwrap()
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Prints the top N trials.
|
||||
fn print_top_trials(study: &Study<f64>, n: usize) {
|
||||
println!("\nTop {n} trials:");
|
||||
|
||||
let mut trials = study.trials();
|
||||
trials.sort_by(|a, b| a.value.partial_cmp(&b.value).unwrap());
|
||||
|
||||
for (i, trial) in trials.iter().take(n).enumerate() {
|
||||
println!(
|
||||
" {}. Trial #{}: score = {:.6}",
|
||||
i + 1,
|
||||
trial.id,
|
||||
trial.value
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Main: Set up and run the async optimization
|
||||
// ============================================================================
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> optimizer::Result<()> {
|
||||
println!("=== Async API Parameter Optimization Example ===\n");
|
||||
|
||||
// Step 1: Create a TPE sampler
|
||||
let sampler = TpeSampler::builder()
|
||||
.n_startup_trials(8)
|
||||
.gamma(0.2)
|
||||
.seed(123)
|
||||
.build()
|
||||
.expect("Failed to build TPE sampler");
|
||||
|
||||
// Step 2: Create a study to minimize the score
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
// Step 3: Define parameter search spaces
|
||||
let cache_size_mb_param = IntParam::new(64, 1024).name("cache_size_mb").step(64);
|
||||
let connection_pool_size_param = IntParam::new(10, 200).name("connection_pool_size").step(10);
|
||||
let request_timeout_ms_param = IntParam::new(1000, 10000)
|
||||
.name("request_timeout_ms")
|
||||
.step(500);
|
||||
let retry_count_param = IntParam::new(0, 5).name("retry_count");
|
||||
let batch_size_param = IntParam::new(1, 256).name("batch_size").log_scale();
|
||||
let compression_level_param = IntParam::new(0, 9).name("compression_level");
|
||||
let use_http2_param = BoolParam::new().name("use_http2");
|
||||
let load_balancing_param = CategoricalParam::new(vec![
|
||||
"round_robin",
|
||||
"least_connections",
|
||||
"random",
|
||||
"ip_hash",
|
||||
])
|
||||
.name("load_balancing");
|
||||
|
||||
// Clone params for use after the closure moves them
|
||||
let cache_size_mb_p = cache_size_mb_param.clone();
|
||||
let connection_pool_size_p = connection_pool_size_param.clone();
|
||||
let request_timeout_ms_p = request_timeout_ms_param.clone();
|
||||
let retry_count_p = retry_count_param.clone();
|
||||
let batch_size_p = batch_size_param.clone();
|
||||
let compression_level_p = compression_level_param.clone();
|
||||
let use_http2_p = use_http2_param.clone();
|
||||
let load_balancing_p = load_balancing_param.clone();
|
||||
|
||||
// Step 4: Configure optimization
|
||||
let n_trials = 40;
|
||||
let concurrency = 4; // Run 4 trials in parallel
|
||||
|
||||
println!("Starting parallel optimization with {concurrency} concurrent evaluations...\n");
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
// Step 5: Run parallel async optimization
|
||||
//
|
||||
// optimize_parallel:
|
||||
// - Runs up to `concurrency` trials simultaneously
|
||||
// - Each trial calls the objective function
|
||||
// - Uses a semaphore to limit concurrent evaluations
|
||||
// - Collects results as trials complete
|
||||
//
|
||||
// The sampler gets access to trial history for informed sampling.
|
||||
study
|
||||
.optimize_parallel(n_trials, concurrency, move |trial| {
|
||||
let cache_size_mb_param = cache_size_mb_param.clone();
|
||||
let connection_pool_size_param = connection_pool_size_param.clone();
|
||||
let request_timeout_ms_param = request_timeout_ms_param.clone();
|
||||
let retry_count_param = retry_count_param.clone();
|
||||
let batch_size_param = batch_size_param.clone();
|
||||
let compression_level_param = compression_level_param.clone();
|
||||
let use_http2_param = use_http2_param.clone();
|
||||
let load_balancing_param = load_balancing_param.clone();
|
||||
async move {
|
||||
objective(
|
||||
trial,
|
||||
&cache_size_mb_param,
|
||||
&connection_pool_size_param,
|
||||
&request_timeout_ms_param,
|
||||
&retry_count_param,
|
||||
&batch_size_param,
|
||||
&compression_level_param,
|
||||
&use_http2_param,
|
||||
&load_balancing_param,
|
||||
)
|
||||
.await
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// Step 5: Print results
|
||||
print_results(&study, elapsed, n_trials);
|
||||
print_best_config(
|
||||
&study,
|
||||
&cache_size_mb_p,
|
||||
&connection_pool_size_p,
|
||||
&request_timeout_ms_p,
|
||||
&retry_count_p,
|
||||
&batch_size_p,
|
||||
&compression_level_p,
|
||||
&use_http2_p,
|
||||
&load_balancing_p,
|
||||
)?;
|
||||
print_top_trials(&study, 5);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
//! Basic optimization example — the "hello world" of the optimizer crate.
|
||||
//!
|
||||
//! Minimizes a simple quadratic function f(x) = (x - 3)² using the default
|
||||
//! random sampler. No feature flags are required.
|
||||
//!
|
||||
//! Run with: `cargo run --example basic_optimization`
|
||||
|
||||
use optimizer::prelude::*;
|
||||
|
||||
fn main() {
|
||||
// Create a study that minimizes the objective function.
|
||||
// The default sampler is random; for smarter sampling, pass a TpeSampler.
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
|
||||
// Search for x in [-10, 10]. The optimizer will suggest values from this range.
|
||||
let x = FloatParam::new(-10.0, 10.0).name("x");
|
||||
|
||||
// Run 50 trials, each evaluating f(x) = (x - 3)²
|
||||
study
|
||||
.optimize(50, |trial| {
|
||||
let x_val = x.suggest(trial)?;
|
||||
let value = (x_val - 3.0).powi(2);
|
||||
Ok::<_, Error>(value)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// Retrieve and display the best result
|
||||
let best = study.best_trial().unwrap();
|
||||
println!("Best trial #{}", best.id);
|
||||
println!(" x = {:.4}", best.get(&x).unwrap());
|
||||
println!(" f(x) = {:.4}", best.value);
|
||||
}
|
||||
@@ -1,124 +0,0 @@
|
||||
use std::ops::ControlFlow;
|
||||
use std::time::Instant;
|
||||
|
||||
use optimizer::parameter::Parameter;
|
||||
use optimizer::sampler::random::RandomSampler;
|
||||
use optimizer::sampler::tpe::TpeSampler;
|
||||
use optimizer::{FloatParam, Study};
|
||||
|
||||
/// Standard optimization test functions.
|
||||
mod functions {
|
||||
pub fn sphere(x: &[f64]) -> f64 {
|
||||
x.iter().map(|xi| xi * xi).sum()
|
||||
}
|
||||
|
||||
pub fn rosenbrock(x: &[f64]) -> f64 {
|
||||
x.windows(2)
|
||||
.map(|w| 100.0 * (w[1] - w[0] * w[0]).powi(2) + (1.0 - w[0]).powi(2))
|
||||
.sum()
|
||||
}
|
||||
|
||||
pub fn rastrigin(x: &[f64]) -> f64 {
|
||||
let n = x.len() as f64;
|
||||
10.0 * n
|
||||
+ x.iter()
|
||||
.map(|xi| xi * xi - 10.0 * (2.0 * std::f64::consts::PI * xi).cos())
|
||||
.sum::<f64>()
|
||||
}
|
||||
}
|
||||
|
||||
fn run_convergence(
|
||||
name: &str,
|
||||
sampler_name: &str,
|
||||
study: Study<f64>,
|
||||
params: &[FloatParam],
|
||||
objective: fn(&[f64]) -> f64,
|
||||
n_trials: usize,
|
||||
) {
|
||||
let start = Instant::now();
|
||||
|
||||
study
|
||||
.optimize_with_callback(
|
||||
n_trials,
|
||||
|trial| {
|
||||
let x: Vec<f64> = params
|
||||
.iter()
|
||||
.map(|p| p.suggest(trial))
|
||||
.collect::<Result<_, _>>()
|
||||
.unwrap();
|
||||
Ok::<_, optimizer::Error>(objective(&x))
|
||||
},
|
||||
|study, _trial| {
|
||||
let elapsed = start.elapsed().as_millis();
|
||||
let best = study.best_value().unwrap();
|
||||
let n = study.n_trials();
|
||||
println!("{n},{best},{elapsed},{sampler_name},{name}");
|
||||
ControlFlow::Continue(())
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn main() {
|
||||
println!("trial,best_value,elapsed_ms,sampler,function");
|
||||
|
||||
let dims = 5;
|
||||
let params: Vec<FloatParam> = (0..dims)
|
||||
.map(|i| FloatParam::new(-5.0, 5.0).name(format!("x{i}")))
|
||||
.collect();
|
||||
let n_trials = 200;
|
||||
|
||||
// Sphere: Random vs TPE
|
||||
run_convergence(
|
||||
"sphere_5d",
|
||||
"random",
|
||||
Study::minimize(RandomSampler::with_seed(1)),
|
||||
¶ms,
|
||||
functions::sphere,
|
||||
n_trials,
|
||||
);
|
||||
run_convergence(
|
||||
"sphere_5d",
|
||||
"tpe",
|
||||
Study::minimize(TpeSampler::builder().seed(1).build().unwrap()),
|
||||
¶ms,
|
||||
functions::sphere,
|
||||
n_trials,
|
||||
);
|
||||
|
||||
// Rosenbrock: Random vs TPE
|
||||
run_convergence(
|
||||
"rosenbrock_5d",
|
||||
"random",
|
||||
Study::minimize(RandomSampler::with_seed(2)),
|
||||
¶ms,
|
||||
functions::rosenbrock,
|
||||
n_trials,
|
||||
);
|
||||
run_convergence(
|
||||
"rosenbrock_5d",
|
||||
"tpe",
|
||||
Study::minimize(TpeSampler::builder().seed(2).build().unwrap()),
|
||||
¶ms,
|
||||
functions::rosenbrock,
|
||||
n_trials,
|
||||
);
|
||||
|
||||
// Rastrigin: Random vs TPE
|
||||
run_convergence(
|
||||
"rastrigin_5d",
|
||||
"random",
|
||||
Study::minimize(RandomSampler::with_seed(3)),
|
||||
¶ms,
|
||||
functions::rastrigin,
|
||||
n_trials,
|
||||
);
|
||||
run_convergence(
|
||||
"rastrigin_5d",
|
||||
"tpe",
|
||||
Study::minimize(TpeSampler::builder().seed(3).build().unwrap()),
|
||||
¶ms,
|
||||
functions::rastrigin,
|
||||
n_trials,
|
||||
);
|
||||
}
|
||||
@@ -1,275 +0,0 @@
|
||||
//! Machine Learning Hyperparameter Tuning Example
|
||||
//!
|
||||
//! This example shows how to use the optimizer library to find the best
|
||||
//! hyperparameters for a machine learning model. We simulate a gradient
|
||||
//! boosting model (like XGBoost or LightGBM) and search for optimal settings.
|
||||
//!
|
||||
//! # Key Concepts Demonstrated
|
||||
//!
|
||||
//! - Creating a Study with a TPE (Tree-Parzen Estimator) sampler
|
||||
//! - Defining an objective function that the optimizer will minimize
|
||||
//! - Using different parameter types: floats, integers, log-scale, stepped
|
||||
//! - Using callbacks to monitor progress and implement early stopping
|
||||
//!
|
||||
//! # How It Works
|
||||
//!
|
||||
//! 1. Create a `Study` - this manages the optimization process
|
||||
//! 2. Define an objective function that takes a `Trial` and returns a score
|
||||
//! 3. Inside the objective, use `trial.suggest_*()` to sample parameters
|
||||
//! 4. The optimizer runs many trials, learning which parameter regions work best
|
||||
//! 5. After optimization, retrieve the best parameters found
|
||||
//!
|
||||
//! Run with: `cargo run --example ml_hyperparameter_tuning`
|
||||
|
||||
use std::ops::ControlFlow;
|
||||
|
||||
use optimizer::prelude::*;
|
||||
|
||||
// ============================================================================
|
||||
// Configuration: Hyperparameters we want to tune
|
||||
// ============================================================================
|
||||
|
||||
/// Holds all the hyperparameters for our model.
|
||||
///
|
||||
/// In a real application, you would pass these to your ML framework
|
||||
/// (e.g., XGBoost, LightGBM, scikit-learn).
|
||||
struct ModelConfig {
|
||||
learning_rate: f64,
|
||||
max_depth: i64,
|
||||
n_estimators: i64,
|
||||
subsample: f64,
|
||||
colsample_bytree: f64,
|
||||
min_child_weight: i64,
|
||||
reg_alpha: f64,
|
||||
reg_lambda: f64,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Objective Function: What we want to optimize
|
||||
// ============================================================================
|
||||
|
||||
/// Simulates training a model and returns the validation loss.
|
||||
///
|
||||
/// In a real scenario, this function would:
|
||||
/// 1. Create a model with the given hyperparameters
|
||||
/// 2. Train it on your training data
|
||||
/// 3. Evaluate it on validation data
|
||||
/// 4. Return the validation metric (e.g., RMSE, log loss, accuracy)
|
||||
///
|
||||
/// The optimizer will try to MINIMIZE this value (we set Direction::Minimize).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn evaluate_model(config: &ModelConfig) -> f64 {
|
||||
// Simulated optimal hyperparameters:
|
||||
// learning_rate ~ 0.05, max_depth ~ 6, n_estimators ~ 200
|
||||
// subsample ~ 0.8, colsample_bytree ~ 0.8, min_child_weight ~ 3
|
||||
// reg_alpha ~ 0.1, reg_lambda ~ 1.0
|
||||
|
||||
let mut loss = 0.15; // Base loss
|
||||
|
||||
// Each term penalizes deviation from the optimal value
|
||||
loss += (config.learning_rate - 0.05).powi(2) * 100.0;
|
||||
loss += ((config.max_depth - 6) as f64).powi(2) * 0.01;
|
||||
loss += ((config.n_estimators - 200) as f64).powi(2) * 0.00001;
|
||||
loss += (config.subsample - 0.8).powi(2) * 10.0;
|
||||
loss += (config.colsample_bytree - 0.8).powi(2) * 10.0;
|
||||
loss += ((config.min_child_weight - 3) as f64).powi(2) * 0.05;
|
||||
loss += (config.reg_alpha - 0.1).powi(2) * 5.0;
|
||||
loss += (config.reg_lambda - 1.0).powi(2) * 2.0;
|
||||
|
||||
// Add some noise to simulate real-world variability
|
||||
let noise = (config.learning_rate * 1000.0).sin() * 0.01;
|
||||
|
||||
loss + noise
|
||||
}
|
||||
|
||||
/// The objective function that the optimizer calls for each trial.
|
||||
///
|
||||
/// This function:
|
||||
/// 1. Uses parameter definitions passed as arguments
|
||||
/// 2. Builds a model configuration from the suggested values
|
||||
/// 3. Evaluates the model and returns the loss
|
||||
///
|
||||
/// The optimizer learns from the results to suggest better parameters
|
||||
/// in future trials.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn objective(
|
||||
trial: &mut Trial,
|
||||
learning_rate_param: &FloatParam,
|
||||
max_depth_param: &IntParam,
|
||||
n_estimators_param: &IntParam,
|
||||
subsample_param: &FloatParam,
|
||||
colsample_bytree_param: &FloatParam,
|
||||
min_child_weight_param: &IntParam,
|
||||
reg_alpha_param: &FloatParam,
|
||||
reg_lambda_param: &FloatParam,
|
||||
) -> optimizer::Result<f64> {
|
||||
let learning_rate = learning_rate_param.suggest(trial)?;
|
||||
let max_depth = max_depth_param.suggest(trial)?;
|
||||
let n_estimators = n_estimators_param.suggest(trial)?;
|
||||
let subsample = subsample_param.suggest(trial)?;
|
||||
let colsample_bytree = colsample_bytree_param.suggest(trial)?;
|
||||
let min_child_weight = min_child_weight_param.suggest(trial)?;
|
||||
let reg_alpha = reg_alpha_param.suggest(trial)?;
|
||||
let reg_lambda = reg_lambda_param.suggest(trial)?;
|
||||
|
||||
// Build configuration and evaluate
|
||||
let config = ModelConfig {
|
||||
learning_rate,
|
||||
max_depth,
|
||||
n_estimators,
|
||||
subsample,
|
||||
colsample_bytree,
|
||||
min_child_weight,
|
||||
reg_alpha,
|
||||
reg_lambda,
|
||||
};
|
||||
|
||||
let loss = evaluate_model(&config);
|
||||
|
||||
Ok(loss)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Callback Function: Monitor progress and implement early stopping
|
||||
// ============================================================================
|
||||
|
||||
/// Called after each successful trial completes.
|
||||
///
|
||||
/// Use callbacks to:
|
||||
/// - Log progress to console or file
|
||||
/// - Save checkpoints
|
||||
/// - Implement early stopping when a good solution is found
|
||||
/// - Track metrics over time
|
||||
///
|
||||
/// Return `ControlFlow::Continue(())` to keep optimizing.
|
||||
/// Return `ControlFlow::Break(())` to stop early.
|
||||
fn on_trial_complete(study: &Study<f64>, trial: &CompletedTrial<f64>) -> ControlFlow<()> {
|
||||
// Print trial number and objective value
|
||||
print!("{:>5} ", study.n_trials());
|
||||
for value in trial.params.values() {
|
||||
print!("{value:>12} ");
|
||||
}
|
||||
println!("{:>12.6}", trial.value);
|
||||
|
||||
// Early stopping: if we find an excellent solution, stop early
|
||||
if trial.value < 0.16 {
|
||||
println!("\nEarly stopping: found excellent solution!");
|
||||
return ControlFlow::Break(());
|
||||
}
|
||||
|
||||
ControlFlow::Continue(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Main: Set up and run the optimization
|
||||
// ============================================================================
|
||||
|
||||
fn main() -> optimizer::Result<()> {
|
||||
println!("=== ML Hyperparameter Tuning Example ===\n");
|
||||
|
||||
// Step 1: Create a sampler
|
||||
//
|
||||
// TPE (Tree-Parzen Estimator) is a Bayesian optimization algorithm.
|
||||
// It learns from previous trials to suggest better parameters.
|
||||
// - n_startup_trials: Number of random trials before TPE kicks in
|
||||
// - gamma: What fraction of trials are considered "good" (lower = more selective)
|
||||
// - seed: For reproducibility
|
||||
let sampler = TpeSampler::builder()
|
||||
.n_startup_trials(10)
|
||||
.gamma(0.25)
|
||||
.seed(42)
|
||||
.build()
|
||||
.expect("Failed to build TPE sampler");
|
||||
|
||||
// Step 2: Create a study
|
||||
//
|
||||
// The study manages the optimization process. We want to MINIMIZE
|
||||
// the loss (lower is better). Use Direction::Maximize for metrics
|
||||
// where higher is better (like accuracy).
|
||||
let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
|
||||
// Print header
|
||||
println!("Starting hyperparameter optimization...\n");
|
||||
println!(
|
||||
"{:>5} {:>12} (parameters...) {:>12}",
|
||||
"Trial", "Params", "Loss"
|
||||
);
|
||||
println!("{}", "-".repeat(60));
|
||||
|
||||
// Step 3: Define parameter search spaces
|
||||
let learning_rate_param = FloatParam::new(0.001, 0.3)
|
||||
.name("learning_rate")
|
||||
.log_scale();
|
||||
let max_depth_param = IntParam::new(3, 12).name("max_depth");
|
||||
let n_estimators_param = IntParam::new(50, 500).name("n_estimators").step(50);
|
||||
let subsample_param = FloatParam::new(0.5, 1.0).name("subsample");
|
||||
let colsample_bytree_param = FloatParam::new(0.5, 1.0).name("colsample_bytree");
|
||||
let min_child_weight_param = IntParam::new(1, 10).name("min_child_weight");
|
||||
let reg_alpha_param = FloatParam::new(1e-3, 10.0).name("reg_alpha").log_scale();
|
||||
let reg_lambda_param = FloatParam::new(1e-3, 10.0).name("reg_lambda").log_scale();
|
||||
|
||||
// Step 4: Run optimization
|
||||
//
|
||||
// optimize_with_callback runs the objective function for up to
|
||||
// n_trials iterations. After each trial, it calls the callback.
|
||||
// The sampler gets access to trial history for informed sampling.
|
||||
let n_trials = 50;
|
||||
|
||||
study.optimize_with_callback(
|
||||
n_trials,
|
||||
|trial| {
|
||||
objective(
|
||||
trial,
|
||||
&learning_rate_param,
|
||||
&max_depth_param,
|
||||
&n_estimators_param,
|
||||
&subsample_param,
|
||||
&colsample_bytree_param,
|
||||
&min_child_weight_param,
|
||||
®_alpha_param,
|
||||
®_lambda_param,
|
||||
)
|
||||
},
|
||||
on_trial_complete,
|
||||
)?;
|
||||
|
||||
// Step 4: Get the best result
|
||||
println!("\n{}", "=".repeat(110));
|
||||
println!("\nOptimization completed!");
|
||||
println!("Total trials: {}", study.n_trials());
|
||||
|
||||
let best = study.best_trial()?;
|
||||
println!("\nBest trial:");
|
||||
println!(" Loss: {:.6}", best.value);
|
||||
println!(" Parameters:");
|
||||
println!(
|
||||
" learning_rate: {:.6}",
|
||||
best.get(&learning_rate_param).unwrap()
|
||||
);
|
||||
println!(" max_depth: {}", best.get(&max_depth_param).unwrap());
|
||||
println!(
|
||||
" n_estimators: {}",
|
||||
best.get(&n_estimators_param).unwrap()
|
||||
);
|
||||
println!(" subsample: {:.6}", best.get(&subsample_param).unwrap());
|
||||
println!(
|
||||
" colsample_bytree: {:.6}",
|
||||
best.get(&colsample_bytree_param).unwrap()
|
||||
);
|
||||
println!(
|
||||
" min_child_weight: {}",
|
||||
best.get(&min_child_weight_param).unwrap()
|
||||
);
|
||||
println!(" reg_alpha: {:.6}", best.get(®_alpha_param).unwrap());
|
||||
println!(
|
||||
" reg_lambda: {:.6}",
|
||||
best.get(®_lambda_param).unwrap()
|
||||
);
|
||||
|
||||
// Step 5: Use the best parameters (in a real app)
|
||||
//
|
||||
// Now you would take best.params and use them to train your final model
|
||||
// on the full dataset.
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
use optimizer::prelude::*;
|
||||
use optimizer_derive::Categorical;
|
||||
|
||||
#[derive(Clone, Debug, Categorical)]
|
||||
enum Activation {
|
||||
Relu,
|
||||
Sigmoid,
|
||||
Tanh,
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
|
||||
// Define parameters outside the objective function
|
||||
let lr_param = FloatParam::new(1e-5, 1e-1).name("lr").log_scale();
|
||||
let n_layers_param = IntParam::new(1, 5).name("n_layers");
|
||||
let units_param = IntParam::new(32, 512).name("units").step(32);
|
||||
let optimizer_param = CategoricalParam::new(vec!["sgd", "adam", "rmsprop"]).name("optimizer");
|
||||
let activation_param = EnumParam::<Activation>::new().name("activation");
|
||||
let batch_size_param = IntParam::new(16, 256).name("batch_size").log_scale();
|
||||
let use_dropout_param = BoolParam::new().name("use_dropout");
|
||||
|
||||
study
|
||||
.optimize(20, |trial| {
|
||||
let lr = lr_param.suggest(trial)?;
|
||||
let n_layers = n_layers_param.suggest(trial)?;
|
||||
let units = units_param.suggest(trial)?;
|
||||
let optimizer = optimizer_param.suggest(trial)?;
|
||||
let use_dropout = use_dropout_param.suggest(trial)?;
|
||||
let activation = activation_param.suggest(trial)?;
|
||||
let batch_size = batch_size_param.suggest(trial)?;
|
||||
|
||||
// Simulate a loss function
|
||||
let loss = lr * (n_layers as f64) + (units as f64) * 0.001
|
||||
- if use_dropout { 0.1 } else { 0.0 };
|
||||
|
||||
println!(
|
||||
"Trial {}: lr={lr:.6}, layers={n_layers}, units={units}, opt={optimizer}, \
|
||||
dropout={use_dropout}, activation={activation:?}, batch={batch_size} -> loss={loss:.4}",
|
||||
trial.id()
|
||||
);
|
||||
|
||||
Ok::<_, Error>(loss)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let best = study.best_trial().unwrap();
|
||||
println!("\nBest trial: value={:.4}", best.value);
|
||||
println!(" lr: {:.6}", best.get(&lr_param).unwrap());
|
||||
println!(" n_layers: {}", best.get(&n_layers_param).unwrap());
|
||||
println!(" units: {}", best.get(&units_param).unwrap());
|
||||
println!(" optimizer: {}", best.get(&optimizer_param).unwrap());
|
||||
println!(" activation: {:?}", best.get(&activation_param).unwrap());
|
||||
println!(" batch_size: {}", best.get(&batch_size_param).unwrap());
|
||||
println!(" use_dropout: {}", best.get(&use_dropout_param).unwrap());
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
//! Parameter types example — demonstrates all five parameter types and the derive feature.
|
||||
//!
|
||||
//! Shows `FloatParam`, `IntParam`, `CategoricalParam`, `BoolParam`, and `EnumParam`
|
||||
//! with `.name()` labels, `#[derive(Categorical)]` for enums, and typed access
|
||||
//! to results via `CompletedTrial::get()`.
|
||||
//!
|
||||
//! Run with: `cargo run --example parameter_types --features derive`
|
||||
|
||||
use optimizer::prelude::*;
|
||||
use optimizer_derive::Categorical;
|
||||
|
||||
/// Activation functions — `#[derive(Categorical)]` auto-generates the
|
||||
/// `Categorical` trait, mapping each variant to a sequential index.
|
||||
#[derive(Clone, Debug, Categorical)]
|
||||
enum Activation {
|
||||
Relu,
|
||||
Sigmoid,
|
||||
Tanh,
|
||||
Gelu,
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||
|
||||
// --- Define one of each parameter type, each with a human-readable .name() ---
|
||||
|
||||
// Float: learning rate on a log scale (common for ML hyperparameters)
|
||||
let lr = FloatParam::new(1e-5, 1e-1).log_scale().name("lr");
|
||||
|
||||
// Int: number of hidden layers (stepped by 1, the default)
|
||||
let n_layers = IntParam::new(1, 5).name("n_layers");
|
||||
|
||||
// Categorical: optimizer algorithm chosen from a list of strings
|
||||
let optimizer = CategoricalParam::new(vec!["sgd", "adam", "rmsprop"]).name("optimizer");
|
||||
|
||||
// Bool: whether to apply dropout
|
||||
let use_dropout = BoolParam::new().name("use_dropout");
|
||||
|
||||
// Enum: activation function — uses #[derive(Categorical)] above
|
||||
let activation = EnumParam::<Activation>::new().name("activation");
|
||||
|
||||
// --- Run the optimization ---
|
||||
study
|
||||
.optimize(30, |trial| {
|
||||
let lr_val = lr.suggest(trial)?;
|
||||
let layers = n_layers.suggest(trial)?;
|
||||
let opt = optimizer.suggest(trial)?;
|
||||
let dropout = use_dropout.suggest(trial)?;
|
||||
let act = activation.suggest(trial)?;
|
||||
|
||||
// Simulated loss that depends on all parameters
|
||||
let loss = lr_val * f64::from(layers as i32)
|
||||
+ if opt == "adam" { -0.05 } else { 0.0 }
|
||||
+ if dropout { -0.02 } else { 0.0 }
|
||||
+ match act {
|
||||
Activation::Gelu => -0.03,
|
||||
Activation::Relu => -0.01,
|
||||
_ => 0.0,
|
||||
};
|
||||
|
||||
Ok::<_, Error>(loss)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
// --- Retrieve best trial and read back each parameter with typed .get() ---
|
||||
let best = study.best_trial().unwrap();
|
||||
println!("Best trial #{} — loss = {:.6}", best.id, best.value);
|
||||
println!(" lr = {:.6}", best.get(&lr).unwrap());
|
||||
println!(" n_layers = {}", best.get(&n_layers).unwrap());
|
||||
println!(" optimizer = {}", best.get(&optimizer).unwrap());
|
||||
println!(" use_dropout = {}", best.get(&use_dropout).unwrap());
|
||||
println!(" activation = {:?}", best.get(&activation).unwrap());
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
//! Pruning and early-stopping example — demonstrates trial pruning with `MedianPruner`
|
||||
//! and early stopping via `optimize_with_callback`.
|
||||
//!
|
||||
//! Simulates a training loop where each trial trains for multiple "epochs". The pruner
|
||||
//! stops unpromising trials early, and a callback halts the entire study once a target
|
||||
//! loss is reached.
|
||||
//!
|
||||
//! Run with: `cargo run --example pruning_and_callbacks`
|
||||
|
||||
use std::ops::ControlFlow;
|
||||
|
||||
use optimizer::TrialState;
|
||||
use optimizer::prelude::*;
|
||||
|
||||
fn main() -> optimizer::Result<()> {
|
||||
let n_trials: usize = 30;
|
||||
let n_epochs: u64 = 20;
|
||||
let target_loss = 0.15;
|
||||
|
||||
// Build a study with a seeded random sampler and MedianPruner.
|
||||
// MedianPruner compares each trial's intermediate value against the median of
|
||||
// completed trials at the same step — trials performing below median are pruned.
|
||||
let study: Study<f64> = Study::builder()
|
||||
.minimize()
|
||||
.sampler(RandomSampler::with_seed(42))
|
||||
.pruner(
|
||||
MedianPruner::new(Direction::Minimize)
|
||||
.n_warmup_steps(3) // let every trial run at least 3 epochs before pruning
|
||||
.n_min_trials(3), // need 3 completed trials before pruning kicks in
|
||||
)
|
||||
.build();
|
||||
|
||||
let learning_rate = FloatParam::new(1e-4, 1.0).name("learning_rate");
|
||||
let momentum = FloatParam::new(0.0, 0.99).name("momentum");
|
||||
|
||||
// Use optimize_with_callback to get both pruning AND early stopping.
|
||||
// The callback fires after each completed (or pruned) trial and can halt the study.
|
||||
study.optimize_with_callback(
|
||||
n_trials,
|
||||
// --- Objective function: simulated training loop with pruning ---
|
||||
|trial| {
|
||||
let lr = learning_rate.suggest(trial)?;
|
||||
let mom = momentum.suggest(trial)?;
|
||||
|
||||
// Simulate training for n_epochs, reporting intermediate loss each epoch.
|
||||
// Good hyperparameters (lr ≈ 0.01, momentum ≈ 0.8) converge to low loss;
|
||||
// bad combos plateau high — giving the pruner something to cut.
|
||||
let mut loss = 1.0;
|
||||
for epoch in 0..n_epochs {
|
||||
let lr_penalty = (lr.log10() - 0.01_f64.log10()).powi(2); // 0 at lr=0.01
|
||||
let mom_penalty = (mom - 0.8).powi(2); // 0 at momentum=0.8
|
||||
let base_loss = 0.02 + 0.05 * lr_penalty + 1.5 * mom_penalty;
|
||||
let progress = (epoch as f64 + 1.0) / n_epochs as f64;
|
||||
// Loss decays from 1.0 toward base_loss over epochs.
|
||||
loss = base_loss + (1.0 - base_loss) * (-3.5 * progress).exp();
|
||||
|
||||
// Report the intermediate value so the pruner can evaluate this trial.
|
||||
trial.report(epoch, loss);
|
||||
|
||||
// Check whether the pruner recommends stopping this trial early.
|
||||
if trial.should_prune() {
|
||||
// Signal that this trial was pruned — the study records it as Pruned.
|
||||
Err(TrialPruned)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok::<_, Error>(loss)
|
||||
},
|
||||
// --- Callback: early stopping when we hit the target ---
|
||||
|study, completed_trial| {
|
||||
let n_complete = study.n_trials();
|
||||
let n_pruned = study
|
||||
.trials()
|
||||
.iter()
|
||||
.filter(|t| t.state == TrialState::Pruned)
|
||||
.count();
|
||||
|
||||
match completed_trial.state {
|
||||
TrialState::Pruned => {
|
||||
println!(
|
||||
" Trial {:>3} PRUNED at epoch {} (loss = {:.4}) \
|
||||
[{n_complete} done, {n_pruned} pruned]",
|
||||
completed_trial.id,
|
||||
completed_trial.intermediate_values.len(),
|
||||
completed_trial
|
||||
.intermediate_values
|
||||
.last()
|
||||
.map_or(f64::NAN, |v| v.1),
|
||||
);
|
||||
}
|
||||
TrialState::Complete => {
|
||||
println!(
|
||||
" Trial {:>3} complete: loss = {:.4} \
|
||||
[{n_complete} done, {n_pruned} pruned]",
|
||||
completed_trial.id, completed_trial.value,
|
||||
);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
// Stop the entire study once we find a good enough result.
|
||||
if completed_trial.state == TrialState::Complete && completed_trial.value < target_loss
|
||||
{
|
||||
println!("\n Early stopping: reached target loss {target_loss}!");
|
||||
return ControlFlow::Break(());
|
||||
}
|
||||
|
||||
ControlFlow::Continue(())
|
||||
},
|
||||
)?;
|
||||
|
||||
// --- Results ---
|
||||
let best = study.best_trial().expect("at least one completed trial");
|
||||
let total = study.n_trials();
|
||||
let pruned = study
|
||||
.trials()
|
||||
.iter()
|
||||
.filter(|t| t.state == TrialState::Pruned)
|
||||
.count();
|
||||
|
||||
println!("\n--- Results ---");
|
||||
println!(" Total trials : {total}");
|
||||
println!(" Pruned : {pruned}");
|
||||
println!(" Completed : {}", total - pruned);
|
||||
println!(" Best trial #{}: loss = {:.6}", best.id, best.value);
|
||||
println!(
|
||||
" learning_rate = {:.6}",
|
||||
best.get(&learning_rate).unwrap()
|
||||
);
|
||||
println!(" momentum = {:.4}", best.get(&momentum).unwrap());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
//! Sampler comparison example — benchmarks Random, TPE, and Grid samplers on the same problem.
|
||||
//!
|
||||
//! Runs the Sphere function f(x, y) = x² + y² with each sampler and compares the best
|
||||
//! value found. This shows how sampler choice affects optimization quality.
|
||||
//!
|
||||
//! Run with: `cargo run --example sampler_comparison`
|
||||
|
||||
use optimizer::prelude::*;
|
||||
|
||||
/// Shared objective function: Sphere function with global minimum at (0, 0).
|
||||
/// Simple enough to solve well, but 2-D so samplers have room to differ.
|
||||
fn sphere(x: f64, y: f64) -> f64 {
|
||||
x.powi(2) + y.powi(2)
|
||||
}
|
||||
|
||||
/// Run an optimization study and return the best value found.
|
||||
fn run_study(study: Study<f64>, n_trials: usize) -> f64 {
|
||||
// Use asymmetric ranges so the Grid sampler tracks each parameter independently.
|
||||
let x = FloatParam::new(-5.0, 5.0).name("x");
|
||||
let y = FloatParam::new(-3.0, 3.0).name("y");
|
||||
|
||||
study
|
||||
.optimize(n_trials, |trial| {
|
||||
let x_val = x.suggest(trial)?;
|
||||
let y_val = y.suggest(trial)?;
|
||||
Ok::<_, Error>(sphere(x_val, y_val))
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let best = study.best_trial().unwrap();
|
||||
println!(
|
||||
" Best trial #{:>3}: x = {:>7.4}, y = {:>7.4}, f(x,y) = {:.6}",
|
||||
best.id,
|
||||
best.get(&x).unwrap(),
|
||||
best.get(&y).unwrap(),
|
||||
best.value,
|
||||
);
|
||||
best.value
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let n_trials: usize = 100;
|
||||
println!("Comparing samplers on Sphere(x, y) = x² + y²");
|
||||
println!(" Search space: x ∈ [-5, 5], y ∈ [-3, 3]");
|
||||
println!(" Known minimum: f(0, 0) = 0");
|
||||
println!(" Trials per sampler: {n_trials}");
|
||||
println!();
|
||||
|
||||
// --- Random sampler (baseline) ---
|
||||
// Pure random search: samples uniformly at random. Fast but not guided.
|
||||
println!("1. Random sampler:");
|
||||
let random_best = run_study(Study::minimize(RandomSampler::with_seed(42)), n_trials);
|
||||
|
||||
// --- TPE sampler (Bayesian) ---
|
||||
// Tree-structured Parzen Estimator: builds a probabilistic model of good vs bad
|
||||
// regions and focuses sampling where improvements are likely.
|
||||
println!("\n2. TPE sampler (Bayesian):");
|
||||
let tpe = TpeSampler::builder()
|
||||
.n_startup_trials(10) // random exploration for the first 10 trials
|
||||
.n_ei_candidates(24) // candidates evaluated per Expected Improvement step
|
||||
.gamma(0.25) // top 25% of trials define the "good" distribution
|
||||
.seed(42)
|
||||
.build()
|
||||
.unwrap();
|
||||
let tpe_best = run_study(Study::minimize(tpe), n_trials);
|
||||
|
||||
// --- Grid sampler (exhaustive) ---
|
||||
// Evaluates evenly spaced grid points. Each parameter gets its own grid that
|
||||
// is sampled in order, so n_points_per_param must be >= n_trials.
|
||||
println!("\n3. Grid sampler (exhaustive):");
|
||||
let grid = GridSearchSampler::builder()
|
||||
.n_points_per_param(n_trials) // one grid point per trial per parameter
|
||||
.build();
|
||||
let grid_best = run_study(Study::minimize(grid), n_trials);
|
||||
|
||||
// --- Summary ---
|
||||
println!("\n--- Summary ---");
|
||||
println!(" Random : {random_best:.6}");
|
||||
println!(" TPE : {tpe_best:.6}");
|
||||
println!(" Grid : {grid_best:.6}");
|
||||
println!();
|
||||
|
||||
// Find the winner
|
||||
let results = [
|
||||
("Random", random_best),
|
||||
("TPE", tpe_best),
|
||||
("Grid", grid_best),
|
||||
];
|
||||
let (winner, _) = results
|
||||
.iter()
|
||||
.min_by(|a, b| a.1.partial_cmp(&b.1).unwrap())
|
||||
.unwrap();
|
||||
println!("Winner: {winner} (closest to known minimum of 0.0)");
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
use optimizer::prelude::*;
|
||||
|
||||
fn main() {
|
||||
// Multi-parameter optimization with TPE sampler.
|
||||
let sampler = TpeSampler::builder().seed(42).build().unwrap();
|
||||
let mut study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler);
|
||||
study.set_pruner(MedianPruner::new(Direction::Minimize));
|
||||
|
||||
let lr = FloatParam::new(1e-5, 1e-1)
|
||||
.log_scale()
|
||||
.name("learning_rate");
|
||||
let n_layers = IntParam::new(1, 5).name("n_layers");
|
||||
let dropout = FloatParam::new(0.0, 0.5).step(0.05).name("dropout");
|
||||
let batch_size = CategoricalParam::new(vec![16, 32, 64, 128]).name("batch_size");
|
||||
|
||||
study
|
||||
.optimize(80, |trial| {
|
||||
let lr_val = lr.suggest(trial)?;
|
||||
let layers = n_layers.suggest(trial)?;
|
||||
let drop = dropout.suggest(trial)?;
|
||||
let bs = batch_size.suggest(trial)?;
|
||||
|
||||
// Simulate training with intermediate reporting.
|
||||
let mut loss = 1.0;
|
||||
for epoch in 0..10 {
|
||||
loss *= 0.7 + 0.3 * lr_val.ln().abs() / 12.0;
|
||||
loss += drop * 0.05;
|
||||
loss += (1.0 / bs as f64) * 0.1;
|
||||
loss -= layers as f64 * 0.02;
|
||||
trial.report(epoch, loss);
|
||||
if trial.should_prune() {
|
||||
return Err(TrialPruned.into());
|
||||
}
|
||||
}
|
||||
|
||||
Ok::<_, Error>(loss)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
println!("{}", study.summary());
|
||||
|
||||
let path = "optimization_report.html";
|
||||
generate_html_report(&study, path).unwrap();
|
||||
println!("\nReport saved to {path}");
|
||||
}
|
||||
Reference in New Issue
Block a user