From 1f1e9d1779713da99ea815a3d1200e21ad7e6548 Mon Sep 17 00:00:00 2001 From: Manuel Raimann Date: Thu, 12 Feb 2026 08:33:23 +0100 Subject: [PATCH] refactor(docs): Documentation Overhaul --- .claude/commands/commit-push-pr.md | 96 +++++++ .claude/commands/commit-push.md | 77 ++++++ .claude/commands/commit.md | 76 ++++++ CHANGELOG.md | 230 ++++++++++++++++ CLAUDE.md | 3 + Cargo.toml | 30 ++- Makefile | 6 + README.md | 174 +++--------- cliff.toml | 99 +++++++ examples/advanced_features.rs | 277 +++++++++++++++++++ examples/async_api_optimization.rs | 368 -------------------------- examples/basic_optimization.rs | 32 +++ examples/benchmark_convergence.rs | 124 --------- examples/ml_hyperparameter_tuning.rs | 275 ------------------- examples/parameter_api.rs | 56 ---- examples/parameter_types.rs | 73 +++++ examples/pruning_and_callbacks.rs | 133 ++++++++++ examples/sampler_comparison.rs | 94 +++++++ examples/visualization_demo.rs | 45 ---- src/error.rs | 60 +++-- src/fanova.rs | 75 +++++- src/importance.rs | 22 ++ src/lib.rs | 213 ++++----------- src/multi_objective.rs | 56 +++- src/param.rs | 33 ++- src/parameter.rs | 152 +++++++---- src/pareto.rs | 103 +++++-- src/pruner/hyperband.rs | 46 +++- src/pruner/median.rs | 36 +++ src/pruner/mod.rs | 38 +++ src/pruner/nop.rs | 12 + src/pruner/patient.rs | 31 +++ src/pruner/percentile.rs | 34 +++ src/pruner/successive_halving.rs | 50 ++++ src/pruner/threshold.rs | 30 +++ src/pruner/wilcoxon.rs | 41 +++ src/sampler/bohb.rs | 58 +++- src/sampler/cma_es.rs | 66 ++++- src/sampler/differential_evolution.rs | 64 ++++- src/sampler/gp.rs | 67 ++++- src/sampler/grid.rs | 74 ++++-- src/sampler/moead.rs | 89 ++++++- src/sampler/motpe.rs | 72 ++++- src/sampler/nsga2.rs | 50 +++- src/sampler/nsga3.rs | 56 +++- src/sampler/random.rs | 41 ++- src/sampler/sobol.rs | 58 +++- src/sampler/tpe/mod.rs | 62 ++++- src/sampler/tpe/multivariate.rs | 52 +++- src/sampler/tpe/sampler.rs | 2 +- src/storage/journal.rs | 106 +++++++- src/storage/memory.rs | 55 +++- src/storage/mod.rs | 49 +++- src/study.rs | 316 ++++++++++++++++------ src/trial.rs | 162 +++++++++--- src/visualization.rs | 56 +++- 56 files changed, 3298 insertions(+), 1557 deletions(-) create mode 100644 .claude/commands/commit-push-pr.md create mode 100644 .claude/commands/commit-push.md create mode 100644 .claude/commands/commit.md create mode 100644 CHANGELOG.md create mode 100644 CLAUDE.md create mode 100644 Makefile create mode 100644 cliff.toml create mode 100644 examples/advanced_features.rs delete mode 100644 examples/async_api_optimization.rs create mode 100644 examples/basic_optimization.rs delete mode 100644 examples/benchmark_convergence.rs delete mode 100644 examples/ml_hyperparameter_tuning.rs delete mode 100644 examples/parameter_api.rs create mode 100644 examples/parameter_types.rs create mode 100644 examples/pruning_and_callbacks.rs create mode 100644 examples/sampler_comparison.rs delete mode 100644 examples/visualization_demo.rs diff --git a/.claude/commands/commit-push-pr.md b/.claude/commands/commit-push-pr.md new file mode 100644 index 0000000..b29a7df --- /dev/null +++ b/.claude/commands/commit-push-pr.md @@ -0,0 +1,96 @@ +--- +allowed-tools: Bash(git add:*), Bash(git status:*), Bash(git commit:*), Bash(git diff:*), Bash(git log:*), Bash(git config:*), Bash(git push:*), Bash(git branch:*), Bash(git rev-parse:*), Bash(gh pr:*) +description: Create a git commit, push, and open a PR +--- + +## Context + +- Current git status: !`git status` +- Current git diff (staged and unstaged changes): !`git diff HEAD` +- Current branch: !`git branch --show-current` +- Remote tracking: !`git rev-parse --abbrev-ref --symbolic-full-name @{u} 2>/dev/null || echo "no upstream"` +- Recent commits (for style reference): !`git log --oneline -10` +- Author commits (for style reference): !`git log --author="$(git config user.email)" --oneline -10` +- All commits on this branch not on main: !`git log --oneline main..HEAD 2>/dev/null` +- User input (optional, can specify target branch): $ARGUMENTS + +## Your task + +Based on the above changes, create a git commit, push, and open a pull request. + +1. If there are no staged changes, stage only the relevant changed files by name (never use `git add -A` or `git add .`). +2. Do NOT commit files that likely contain secrets (.env, credentials.json, etc). +3. Write a commit message following the guidelines below, then commit. +4. Push to the remote. Use `git push -u origin ` if there is no upstream set. +5. Create a pull request using `gh pr create` targeting the base branch: + - If the user specified a target branch in their input, use that. + - Otherwise, default to `main`. +6. The PR title should match the commit title (or summarize all branch commits if multiple). +7. Write the PR body following the PR format below. +8. Return the PR URL when done. + +## Commit message guidelines + +- Use imperative mood (e.g., "add feature" not "added feature") +- First line: brief summary, max 72 characters +- Focus on the "why" and "what", not the "how" +- Be specific but concise +- Many commits do not need a body if the title is self-explanatory + - Litmus test: "Would a developer understand this commit from the title + diff?" If yes, skip the body. +- Do NOT include "Generated with ...", "Co-Authored-By ...", or any AI attribution + +### Conventional commit prefixes + +Match the prefix to the nature of the change. These must align with `cliff.toml` commit parsers so they appear correctly in the changelog. + +**Changelog: "Added"** +- `feat:` / `feat(scope):` — new feature + +**Changelog: "Fixed"** +- `fix:` / `fix(scope):` — bug fix + +**Changelog: "Changed"** +- `refactor:` / `refactor(scope):` — code restructuring without behavior change +- `perf:` / `perf(scope):` — performance improvement +- `docs:` / `docs(scope):` — documentation changes (README, guides, etc.) +- `style:` / `style(scope):` — formatting only (no logic change) +- `chore:` / `chore(scope):` — tooling, deps, config, scripts, AI config (.claude/, CLAUDE.md) + +**Changelog: "Removed"** +- `revert:` / `revert(scope):` — revert a previous commit + +**Excluded from changelog** +- `test:` / `test(scope):` — adding or updating tests +- `ci:` / `ci(scope):` — CI/CD pipeline changes + +### Body rules + +- When a body is needed (multiple important things in one commit), use bullet points. +- Body is separated from the title by a blank line. +- One bullet point per concept/change. +- Don't explain obvious things like "added unit tests for X". + +### Commit format + +Always pass the commit message via a HEREDOC: + +``` +git commit -m "$(cat <<'EOF' + + +<optional body> +EOF +)" +``` + +## PR format + +Use a HEREDOC for the body. Look at ALL commits on the branch (not just the latest) to write the summary. + +``` +gh pr create --title "<title>" --base <target-branch> --body "$(cat <<'EOF' +## Summary +<1-3 bullet points covering all branch commits> +EOF +)" +``` diff --git a/.claude/commands/commit-push.md b/.claude/commands/commit-push.md new file mode 100644 index 0000000..a3f38c7 --- /dev/null +++ b/.claude/commands/commit-push.md @@ -0,0 +1,77 @@ +--- +allowed-tools: Bash(git add:*), Bash(git status:*), Bash(git commit:*), Bash(git diff:*), Bash(git log:*), Bash(git config:*), Bash(git push:*), Bash(git branch:*), Bash(git rev-parse:*) +description: Create a git commit and push to remote +--- + +## Context + +- Current git status: !`git status` +- Current git diff (staged and unstaged changes): !`git diff HEAD` +- Current branch: !`git branch --show-current` +- Remote tracking: !`git rev-parse --abbrev-ref --symbolic-full-name @{u} 2>/dev/null || echo "no upstream"` +- Recent commits (for style reference): !`git log --oneline -10` +- Author commits (for style reference): !`git log --author="$(git config user.email)" --oneline -10` +- User input (optional): $ARGUMENTS + +## Your task + +Based on the above changes, create a single git commit and push it to the remote. + +1. If there are no staged changes, stage only the relevant changed files by name (never use `git add -A` or `git add .`). +2. Do NOT commit files that likely contain secrets (.env, credentials.json, etc). +3. Write a commit message following the guidelines below, then commit. +4. Push to the remote. Use `git push -u origin <branch>` if there is no upstream set. + +## Commit message guidelines + +- Use imperative mood (e.g., "add feature" not "added feature") +- First line: brief summary, max 72 characters +- Focus on the "why" and "what", not the "how" +- Be specific but concise +- Many commits do not need a body if the title is self-explanatory + - Litmus test: "Would a developer understand this commit from the title + diff?" If yes, skip the body. +- Do NOT include "Generated with ...", "Co-Authored-By ...", or any AI attribution + +### Conventional commit prefixes + +Match the prefix to the nature of the change. These must align with `cliff.toml` commit parsers so they appear correctly in the changelog. + +**Changelog: "Added"** +- `feat:` / `feat(scope):` — new feature + +**Changelog: "Fixed"** +- `fix:` / `fix(scope):` — bug fix + +**Changelog: "Changed"** +- `refactor:` / `refactor(scope):` — code restructuring without behavior change +- `perf:` / `perf(scope):` — performance improvement +- `docs:` / `docs(scope):` — documentation changes (README, guides, etc.) +- `style:` / `style(scope):` — formatting only (no logic change) +- `chore:` / `chore(scope):` — tooling, deps, config, scripts, AI config (.claude/, CLAUDE.md) + +**Changelog: "Removed"** +- `revert:` / `revert(scope):` — revert a previous commit + +**Excluded from changelog** +- `test:` / `test(scope):` — adding or updating tests +- `ci:` / `ci(scope):` — CI/CD pipeline changes + +### Body rules + +- When a body is needed (multiple important things in one commit), use bullet points. +- Body is separated from the title by a blank line. +- One bullet point per concept/change. +- Don't explain obvious things like "added unit tests for X". + +### Commit format + +Always pass the commit message via a HEREDOC: + +``` +git commit -m "$(cat <<'EOF' +<title line> + +<optional body> +EOF +)" +``` diff --git a/.claude/commands/commit.md b/.claude/commands/commit.md new file mode 100644 index 0000000..50d3362 --- /dev/null +++ b/.claude/commands/commit.md @@ -0,0 +1,76 @@ +--- +allowed-tools: Bash(git add:*), Bash(git status:*), Bash(git commit:*), Bash(git diff:*), Bash(git log:*), Bash(git config:*), Bash(git branch:*) +description: Create a git commit +--- + +## Context + +- Current git status: !`git status` +- Current git diff (staged and unstaged changes): !`git diff HEAD` +- Current branch: !`git branch --show-current` +- Recent commits (for style reference): !`git log --oneline -10` +- Author commits (for style reference): !`git log --author="$(git config user.email)" --oneline -10` +- User input (optional): $ARGUMENTS + +## Your task + +Based on the above changes, create a single git commit. + +1. If there are no staged changes, stage only the relevant changed files by name (never use `git add -A` or `git add .`). +2. Do NOT commit files that likely contain secrets (.env, credentials.json, etc). +3. Write a commit message following the guidelines below, then commit. +4. Do NOT push to remote. + +## Commit message guidelines + +- Use imperative mood (e.g., "add feature" not "added feature") +- First line: brief summary, max 72 characters +- Focus on the "why" and "what", not the "how" +- Be specific but concise +- Many commits do not need a body if the title is self-explanatory + - Litmus test: "Would a developer understand this commit from the title + diff?" If yes, skip the body. +- Do NOT include "Generated with ...", "Co-Authored-By ...", or any AI attribution + +### Conventional commit prefixes + +Match the prefix to the nature of the change. These must align with `cliff.toml` commit parsers so they appear correctly in the changelog. + +**Changelog: "Added"** +- `feat:` / `feat(scope):` — new feature + +**Changelog: "Fixed"** +- `fix:` / `fix(scope):` — bug fix + +**Changelog: "Changed"** +- `refactor:` / `refactor(scope):` — code restructuring without behavior change +- `perf:` / `perf(scope):` — performance improvement +- `docs:` / `docs(scope):` — documentation changes (README, guides, etc.) +- `style:` / `style(scope):` — formatting only (no logic change) +- `chore:` / `chore(scope):` — tooling, deps, config, scripts, AI config (.claude/, CLAUDE.md) + +**Changelog: "Removed"** +- `revert:` / `revert(scope):` — revert a previous commit + +**Excluded from changelog** +- `test:` / `test(scope):` — adding or updating tests +- `ci:` / `ci(scope):` — CI/CD pipeline changes + +### Body rules + +- When a body is needed (multiple important things in one commit), use bullet points. +- Body is separated from the title by a blank line. +- One bullet point per concept/change. +- Don't explain obvious things like "added unit tests for X". + +### Commit format + +Always pass the commit message via a HEREDOC: + +``` +git commit -m "$(cat <<'EOF' +<title line> + +<optional body> +EOF +)" +``` diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..2b0acf9 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,230 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.9.1] - 2026-02-12 + +### Fixed + +- Fix journal file handling to support both reading and writing modes + +## [0.9.0] - 2026-02-12 + +### Added + +- Storage trait abstraction with pluggable backends +- JSONL journal storage backend for persistent studies (behind `journal` feature) +- Differential Evolution sampler +- Gaussian Process sampler with Expected Improvement acquisition function +- NSGA-III sampler for many-objective optimization with reference-point decomposition +- MOEA/D sampler for decomposition-based multi-objective optimization +- `StudyBuilder` for fluent study construction + +### Changed + +- Move `next_trial_id` counter from `Study` into `Storage` trait +- Replace `rand 0.10` with `fastrand 2.3` for simpler, faster random number generation +- Remove `visualization` and `fanova` feature flags (now always available) +- Simplify CI test commands by removing feature matrix + +### Fixed + +- Seed importance tests to eliminate flakiness +- Use atomic counter for temp paths in journal tests + +### Removed + +- Reverted SQLite storage backend (may return in a future release) + +## [0.8.1] - 2026-02-11 + +### Fixed + +- Resolve broken rustdoc intra-doc links for `Error::NoCompletedTrials` + +## [0.8.0] - 2026-02-11 + +### Added + +- Multi-objective optimization with NSGA-II +- Multi-Objective TPE (MOTPE) sampler +- Pareto front analysis utilities +- CSV and JSON data export for visualization +- HTML visualization reports with Plotly.js charts +- fANOVA (functional ANOVA) parameter importance via random forest + +### Fixed + +- Rename variable to pass typos check + +## [0.7.2] - 2026-02-11 + +### Changed + +- Update `nalgebra` dependency to version 0.34 +- Add advisory ignore for unmaintained transitive dependency + +## [0.7.1] - 2026-02-11 + +### Changed + +- Update `tracing` dependency to version 0.1.29 + +## [0.7.0] - 2026-02-11 + +### Added + +- CMA-ES (Covariance Matrix Adaptation Evolution Strategy) sampler behind `cma-es` feature flag +- Sobol quasi-random sampler behind `sobol` feature flag +- BOHB (Bayesian Optimization with HyperBand) sampler for budget-aware optimization +- Parameter importance analysis via Spearman rank correlation +- Constraint handling with feasibility-aware trial ranking +- `optimize_with_retries()` for automatic retry of failed trials +- `optimize_with_checkpoint()` with atomic save writes for crash recovery +- `Study::summary()` and `Display` impl for study overview +- `IntoIterator` for `&Study` and `iter()` method +- Tracing integration behind `tracing` feature flag +- Serde serialization support behind `serde` feature flag +- Benchmark suite with criterion and standard test functions + +## [0.6.0] - 2026-02-11 + +### Added + +- Pruning system with `Pruner` trait and `NopPruner` default implementation +- Intermediate value reporting on trials for pruner integration +- `TrialPruned` error variant and `Pruned` trial state +- `ThresholdPruner` for fixed-bound trial pruning +- `MedianPruner` for statistics-based trial pruning +- `PercentilePruner` for configurable percentile-based trial pruning +- `PatientPruner` for patience-based trial pruning +- `SuccessiveHalvingPruner` for SHA-based trial pruning +- `HyperbandPruner` for multi-bracket trial pruning +- `WilcoxonPruner` for statistics-based trial pruning +- `Study::minimize()` and `Study::maximize()` constructor shortcuts +- `From<RangeInclusive>` for `FloatParam` and `IntParam` +- Timeout-based optimization with `optimize_until` +- `Study::top_trials(n)` for retrieving the best N trials +- Ask-and-tell interface with `ask()` and `tell()` methods +- Trial user attributes for logging and analysis +- `enqueue_trial()` for pre-specified parameter evaluation + +## [0.5.1] - 2026-02-10 + +### Changed + +- Update random number generation to use `rand::make_rng()` and upgrade `rand` to version 0.10 + +## [0.5.0] - 2026-02-06 + +### Added + +- Typed parameter API with `FloatParam`, `IntParam`, `CategoricalParam`, `BoolParam`, and `EnumParam` +- `#[derive(Categorical)]` proc macro for deriving categorical parameters from enums (behind `derive` feature) +- `.name()` builder method on all parameter types for custom labels +- `CompletedTrial::get()` for typed parameter access +- `Display` impl on `ParamValue` +- Prelude module at `optimizer::prelude::*` + +### Changed + +- Reorganize sampler module structure and update imports +- Remove `log` dependency + +## [0.4.0] - 2026-02-02 + +### Added + +- Multivariate TPE sampler for correlated parameter search +- Gamma strategies for TPE sampler (linear, sqrt, fixed) with examples +- Example: async API parameter optimization +- Example: ML hyperparameter tuning + +### Fixed + +- Handle end value in `suggest` method to avoid panic on underflow + +## [0.3.1] - 2026-01-31 + +### Added + +- Grid search sampler for exhaustive parameter exploration +- `suggest_bool()` method for boolean parameter suggestion +- `SuggestableRange` trait and `suggest_range()` method for parameter suggestion from ranges +- Documentation, keywords, categories, and readme fields in `Cargo.toml` + +### Changed + +- Replace `TpeError` with unified `Error` type across the library +- Add permissions for read access to contents in CI and scheduled workflows + +## [0.3.0] - 2026-01-30 + +### Added + +- Cross-target compilation checks in CI + +### Changed + +- Internal code refactoring and cleanup + +## [0.2.0] - 2026-01-30 + +### Added + +- CI step for unused dependencies check with `cargo-machete` +- CI step to publish to crates.io +- Default typo extension for 'Tpe' + +### Changed + +- Remove `serde` dependency (was unused) +- Remove unused `ordered-float` dependency + +## [0.1.1] - 2026-01-30 + +### Fixed + +- Minor release fixes + +## [0.1.0] - 2026-01-30 + +### Added + +- Initial implementation of the optimization library +- TPE (Tree-structured Parzen Estimator) sampler with optional fixed bandwidth KDE +- Random sampler +- Async optimization support +- CI workflows for testing, coverage (Codecov), auditing, and publishing +- README with project overview, features, and quick start guide + +--- + +## Release Workflow + +This changelog is maintained automatically with [git-cliff](https://git-cliff.org/). + +1. Write commits using [Conventional Commits](https://www.conventionalcommits.org/) (`feat:`, `fix:`, `refactor:`, etc.). +2. Tag the release: `git tag v0.X.0` +3. Regenerate the changelog: `make changelog` +4. Commit the updated `CHANGELOG.md` and push. + +[0.9.1]: https://github.com/raimannma/rust-optimizer/compare/v0.9.0...v0.9.1 +[0.9.0]: https://github.com/raimannma/rust-optimizer/compare/v0.8.1...v0.9.0 +[0.8.1]: https://github.com/raimannma/rust-optimizer/compare/v0.8.0...v0.8.1 +[0.8.0]: https://github.com/raimannma/rust-optimizer/compare/v0.7.2...v0.8.0 +[0.7.2]: https://github.com/raimannma/rust-optimizer/compare/v0.7.1...v0.7.2 +[0.7.1]: https://github.com/raimannma/rust-optimizer/compare/v0.7.0...v0.7.1 +[0.7.0]: https://github.com/raimannma/rust-optimizer/compare/v0.6.0...v0.7.0 +[0.6.0]: https://github.com/raimannma/rust-optimizer/compare/v0.5.1...v0.6.0 +[0.5.1]: https://github.com/raimannma/rust-optimizer/compare/v0.5.0...v0.5.1 +[0.5.0]: https://github.com/raimannma/rust-optimizer/compare/v0.4.0...v0.5.0 +[0.4.0]: https://github.com/raimannma/rust-optimizer/compare/v0.3.1...v0.4.0 +[0.3.1]: https://github.com/raimannma/rust-optimizer/compare/v0.3.0...v0.3.1 +[0.3.0]: https://github.com/raimannma/rust-optimizer/compare/v0.2.0...v0.3.0 +[0.2.0]: https://github.com/raimannma/rust-optimizer/compare/v0.1.1...v0.2.0 +[0.1.1]: https://github.com/raimannma/rust-optimizer/compare/v0.1.0...v0.1.1 +[0.1.0]: https://github.com/raimannma/rust-optimizer/releases/tag/v0.1.0 diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..fa93e9d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,3 @@ +## Git Guidelines + +- If the user asks to make a commit, do not add any co-author by claude or any other co-author note. diff --git a/Cargo.toml b/Cargo.toml index 0d45868..d1c11d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,11 +8,11 @@ edition = "2024" rust-version = "1.89" license = "MIT" authors = ["Manuel Raimann <raimannma@outlook.de"] -description = "A Rust library for optimization algorithms." +description = "Bayesian and population-based optimization library with an Optuna-like API for hyperparameter tuning and black-box optimization" repository = "https://github.com/raimannma/rust-optimizer" documentation = "https://docs.rs/optimizer" keywords = ["optimization", "hyperparameter", "tpe", "grid-search", "bayesian"] -categories = ["algorithm", "science", "data-structures"] +categories = ["algorithms", "science", "mathematics"] readme = "README.md" [dependencies] @@ -54,22 +54,26 @@ name = "optimization" harness = false [[example]] -name = "async_api_optimization" -path = "examples/async_api_optimization.rs" -required-features = ["async"] +name = "basic_optimization" +path = "examples/basic_optimization.rs" [[example]] -name = "ml_hyperparameter_tuning" -path = "examples/ml_hyperparameter_tuning.rs" - -[[example]] -name = "parameter_api" -path = "examples/parameter_api.rs" +name = "parameter_types" +path = "examples/parameter_types.rs" required-features = ["derive"] [[example]] -name = "visualization_demo" -path = "examples/visualization_demo.rs" +name = "sampler_comparison" +path = "examples/sampler_comparison.rs" + +[[example]] +name = "pruning_and_callbacks" +path = "examples/pruning_and_callbacks.rs" + +[[example]] +name = "advanced_features" +path = "examples/advanced_features.rs" +required-features = ["async", "journal"] [[test]] name = "journal_tests" diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d1a6f66 --- /dev/null +++ b/Makefile @@ -0,0 +1,6 @@ +.PHONY: changelog + +# Regenerate CHANGELOG.md from git tags using git-cliff. +# Install: cargo install git-cliff +changelog: + git-cliff --output CHANGELOG.md diff --git a/README.md b/README.md index 56cc9f1..970013c 100644 --- a/README.md +++ b/README.md @@ -1,155 +1,69 @@ # optimizer -A Rust library for black-box optimization with multiple sampling strategies. +Bayesian and population-based optimization library with an Optuna-like API +for hyperparameter tuning and black-box optimization. Supports 12 samplers, +8 pruners, multi-objective optimization, async parallelism, and persistent storage. [![Docs](https://docs.rs/optimizer/badge.svg)](https://docs.rs/optimizer) [![Crates.io](https://img.shields.io/crates/v/optimizer.svg)](https://crates.io/crates/optimizer) [![codecov](https://codecov.io/gh/raimannma/rust-optimizer/graph/badge.svg?token=WOE77XJ4M6)](https://codecov.io/gh/raimannma/rust-optimizer) -## Features - -- Optuna-like API for hyperparameter optimization -- Multiple sampling strategies: - - **Random Search** - Simple random sampling for baseline comparisons - - **TPE (Tree-Parzen Estimator)** - Bayesian optimization for efficient search - - **Grid Search** - Exhaustive search over a specified parameter grid -- Float, integer, categorical, boolean, and enum parameter types -- Log-scale and stepped parameter sampling -- Sync and async optimization with parallel trial evaluation -- `#[derive(Categorical)]` for enum parameters - ## Quick Start ```rust -use optimizer::parameter::{FloatParam, Parameter}; -use optimizer::sampler::tpe::TpeSampler; -use optimizer::{Direction, Study}; +use optimizer::prelude::*; -// Create a study with TPE sampler -let sampler = TpeSampler::builder().seed(42).build().unwrap(); -let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler); +let study: Study<f64> = Study::new(Direction::Minimize); +let x = FloatParam::new(-10.0, 10.0).name("x"); -// Define parameter search space -let x_param = FloatParam::new(-10.0, 10.0); +study.optimize(50, |trial| { + let val = x.suggest(trial)?; + Ok::<_, Error>((val - 3.0).powi(2)) +}).unwrap(); -// Optimize x^2 for 20 trials -study - .optimize_with_sampler(20, |trial| { - let x = x_param.suggest(trial)?; - Ok::<_, optimizer::Error>(x * x) - }) - .unwrap(); - -// Get the best result let best = study.best_trial().unwrap(); -println!("Best value: {}", best.value); -for (id, label) in &best.param_labels { - println!(" {}: {:?}", label, best.params[id]); -} +println!("Best x = {:.4}, f(x) = {:.4}", best.get(&x).unwrap(), best.value); ``` -## Samplers +## Features at a Glance -### Random Search - -```rust -use optimizer::{Direction, Study}; -use optimizer::sampler::random::RandomSampler; - -let study: Study<f64> = Study::with_sampler( - Direction::Minimize, - RandomSampler::with_seed(42), -); -``` - -### TPE (Tree-Parzen Estimator) - -```rust -use optimizer::{Direction, Study}; -use optimizer::sampler::tpe::TpeSampler; - -let sampler = TpeSampler::builder() - .gamma(0.15) // Quantile for good/bad split - .n_startup_trials(20) // Random trials before TPE kicks in - .n_ei_candidates(32) // Candidates to evaluate - .seed(42) - .build() - .unwrap(); - -let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler); -``` - -#### Gamma Strategies - -The gamma parameter controls what fraction of trials are considered "good" when building the TPE model. Instead of a fixed value, you can use adaptive strategies: - -| Strategy | Description | Formula | -|----------|-------------|---------| -| `FixedGamma` | Constant value (default: 0.25) | `γ = constant` | -| `LinearGamma` | Linear interpolation over trials | `γ = γ_min + (γ_max - γ_min) * min(n/n_max, 1)` | -| `SqrtGamma` | Optuna-style inverse sqrt scaling | `γ = min(γ_max, factor/√n / n)` | -| `HyperoptGamma` | Hyperopt-style adaptive | `γ = min(γ_max, (base + 1) / n)` | - -```rust -use optimizer::sampler::tpe::{TpeSampler, SqrtGamma, LinearGamma}; - -// Optuna-style gamma that decreases with more trials -let sampler = TpeSampler::builder() - .gamma_strategy(SqrtGamma::default()) - .build() - .unwrap(); - -// Linear interpolation from 0.1 to 0.3 over 100 trials -let sampler = TpeSampler::builder() - .gamma_strategy(LinearGamma::new(0.1, 0.3, 100).unwrap()) - .build() - .unwrap(); -``` - -You can also implement custom strategies: - -```rust -use optimizer::sampler::tpe::{TpeSampler, GammaStrategy}; - -#[derive(Debug, Clone)] -struct MyGamma { base: f64 } - -impl GammaStrategy for MyGamma { - fn gamma(&self, n_trials: usize) -> f64 { - (self.base + 0.01 * n_trials as f64).min(0.5) - } - fn clone_box(&self) -> Box<dyn GammaStrategy> { - Box::new(self.clone()) - } -} - -let sampler = TpeSampler::builder() - .gamma_strategy(MyGamma { base: 0.1 }) - .build() - .unwrap(); -``` - -### Grid Search - -```rust -use optimizer::{Direction, Study}; -use optimizer::sampler::grid::GridSearchSampler; - -let sampler = GridSearchSampler::builder() - .n_points_per_param(10) // Number of points per parameter dimension - .build(); - -let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler); -``` +- **[Samplers](https://docs.rs/optimizer/latest/optimizer/sampler/)** — Random, TPE, Multivariate TPE, Grid, Sobol, CMA-ES, Gaussian Process, Differential Evolution, BOHB, NSGA-II, NSGA-III, MOEA/D +- **[Pruners](https://docs.rs/optimizer/latest/optimizer/pruner/)** — Median, Percentile, Threshold, Patient, Hyperband, Successive Halving, Wilcoxon, Nop +- **[Parameters](https://docs.rs/optimizer/latest/optimizer/parameter/)** — Float, Int, Categorical, Bool, and Enum types with `.name()` labels and typed access +- **[Multi-objective](https://docs.rs/optimizer/latest/optimizer/multi_objective/)** — Pareto front extraction with NSGA-II/III and MOEA/D +- **[Async & parallel](https://docs.rs/optimizer/latest/optimizer/struct.Study.html#method.optimize_parallel)** — Concurrent trial evaluation with Tokio +- **[Storage backends](https://docs.rs/optimizer/latest/optimizer/storage/)** — In-memory (default) or JSONL journal for persistence and resumption +- **[Visualization](https://docs.rs/optimizer/latest/optimizer/fn.generate_html_report.html)** — HTML reports with optimization history, parameter importance, and Pareto fronts +- **[Analysis](https://docs.rs/optimizer/latest/optimizer/struct.Study.html#method.fanova)** — fANOVA and Spearman correlation for parameter importance ## Feature Flags -- `async` - Enable async optimization methods (requires tokio) -- `derive` - Enable `#[derive(Categorical)]` for enum parameters +| Flag | Enables | Default | +|------|---------|---------| +| `async` | Async/parallel optimization (Tokio) | No | +| `derive` | `#[derive(Categorical)]` for enum parameters | No | +| `serde` | Serialization of trials and parameters | No | +| `journal` | JSONL storage backend (implies `serde`) | No | +| `sobol` | Sobol quasi-random sampler | No | +| `cma-es` | CMA-ES sampler (requires `nalgebra`) | No | +| `gp` | Gaussian Process sampler (requires `nalgebra`) | No | +| `tracing` | Structured logging with `tracing` | No | -## Documentation +## Examples -Full API documentation is available at [docs.rs/optimizer](https://docs.rs/optimizer). +```sh +cargo run --example basic_optimization # Minimize a quadratic — simplest possible usage +cargo run --example sampler_comparison # Compare Random, TPE, and Grid on the same problem +cargo run --example pruning_and_callbacks # Trial pruning with MedianPruner + early stopping +cargo run --example parameter_types --features derive # All 5 param types + #[derive(Categorical)] +cargo run --example advanced_features --features async,journal # Async, journal storage, ask-and-tell, multi-objective +``` + +## Learn More + +- [API documentation](https://docs.rs/optimizer) +- [Changelog](CHANGELOG.md) +- [GitHub Issues](https://github.com/raimannma/rust-optimizer/issues) ## License diff --git a/cliff.toml b/cliff.toml new file mode 100644 index 0000000..c2166f6 --- /dev/null +++ b/cliff.toml @@ -0,0 +1,99 @@ +# git-cliff configuration +# https://git-cliff.org/docs/configuration + +[changelog] +# changelog header +header = """ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).\n +""" +# template for the changelog body +# https://keats.github.io/tera/docs/#introduction +body = """ +{%- macro remote_url() -%} + https://github.com/raimannma/rust-optimizer +{%- endmacro -%} + +{% if version -%} + ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }} +{% else -%} + ## [Unreleased] +{% endif -%} + +{% for group, commits in commits | group_by(attribute="group") %} + ### {{ group | striptags | trim }} + {% for commit in commits %} + - {{ commit.message | split(pat="\n") | first | trim }}\ + {%- if commit.breaking %} (**BREAKING**){% endif -%} + {% endfor %} +{% endfor %} +""" +# template for the changelog footer +footer = """ +{%- macro remote_url() -%} + https://github.com/raimannma/rust-optimizer +{%- endmacro -%} + +{% for release in releases -%} + {% if release.version -%} + {% if release.previous.version -%} + [{{ release.version | trim_start_matches(pat="v") }}]: \ + {{ self::remote_url() }}/compare/{{ release.previous.version }}...{{ release.version }} + {% else -%} + [{{ release.version | trim_start_matches(pat="v") }}]: \ + {{ self::remote_url() }}/releases/tag/{{ release.version }} + {% endif -%} + {% else -%} + {% if release.previous.version -%} + [Unreleased]: {{ self::remote_url() }}/compare/{{ release.previous.version }}...HEAD + {% endif -%} + {% endif -%} +{% endfor %} +""" +# remove the leading and trailing whitespace from the templates +trim = true + +[git] +# parse the commits based on https://www.conventionalcommits.org +conventional_commits = true +# filter out the commits that are not conventional +filter_unconventional = false +# process each line of a commit as an individual commit +split_commits = false +# regex for preprocessing the commit messages +commit_preprocessors = [ + { pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](https://github.com/raimannma/rust-optimizer/issues/${2}))" }, +] +# regex for parsing and grouping commits +commit_parsers = [ + { message = "^feat", group = "Added" }, + { message = "^fix", group = "Fixed" }, + { message = "^refactor", group = "Changed" }, + { message = "^perf", group = "Changed" }, + { message = "^doc", group = "Changed" }, + { message = "^style", group = "Changed" }, + { message = "^ci", skip = true }, + { message = "^chore\\(release\\)", skip = true }, + { message = "^chore: release", skip = true }, + { message = "^chore", group = "Changed" }, + { message = "^revert", group = "Removed" }, + { body = ".*security", group = "Security" }, +] +# protect breaking changes from being skipped due to matching a skipping commit_parser +protect_breaking_commits = true +# filter out the commits that are not matched by commit parsers +filter_commits = true +# regex for matching git tags +tag_pattern = "v[0-9].*" +# regex for skipping tags +skip_tags = "" +# regex for ignoring tags +ignore_tags = "" +# sort the tags topologically +topo_order = false +# sort the commits inside sections by oldest first +sort_commits = "oldest" diff --git a/examples/advanced_features.rs b/examples/advanced_features.rs new file mode 100644 index 0000000..41eed01 --- /dev/null +++ b/examples/advanced_features.rs @@ -0,0 +1,277 @@ +//! Advanced Features Example +//! +//! This example demonstrates four advanced capabilities of the optimizer crate: +//! +//! 1. **Async parallel optimization** — evaluate multiple trials concurrently +//! 2. **Journal storage** — persist trials to disk and resume studies later +//! 3. **Ask-and-tell interface** — decouple sampling from evaluation +//! 4. **Multi-objective optimization** — optimize competing objectives simultaneously +//! +//! Run with: `cargo run --example advanced_features --features "async,journal"` + +use std::time::Instant; + +use optimizer::multi_objective::MultiObjectiveStudy; +use optimizer::prelude::*; + +// ============================================================================ +// Section 1: Async Parallel Optimization +// ============================================================================ + +/// Runs multiple trials concurrently using tokio, reducing wall-clock time +/// when the objective function involves I/O or other async work. +async fn async_parallel_optimization() -> optimizer::Result<()> { + println!("=== Section 1: Async Parallel Optimization ===\n"); + + let sampler = TpeSampler::builder() + .n_startup_trials(5) + .seed(42) + .build() + .expect("Failed to build TPE sampler"); + + let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler); + + let x = FloatParam::new(-5.0, 5.0).name("x"); + let y = FloatParam::new(-5.0, 5.0).name("y"); + + let n_trials = 30; + let concurrency = 4; + + println!("Running {n_trials} trials with {concurrency} concurrent workers..."); + let start = Instant::now(); + + // optimize_parallel spawns up to `concurrency` trials at once. + // The closure must take ownership of Trial and return (Trial, value). + study + .optimize_parallel(n_trials, concurrency, { + let x = x.clone(); + let y = y.clone(); + move |mut trial| { + let x = x.clone(); + let y = y.clone(); + async move { + let xv = x.suggest(&mut trial)?; + let yv = y.suggest(&mut trial)?; + + // Simulate async I/O (e.g., calling an external service) + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + + // Sphere function: minimum at origin + let value = xv * xv + yv * yv; + Ok::<_, optimizer::Error>((trial, value)) + } + } + }) + .await?; + + let elapsed = start.elapsed(); + let best = study.best_trial()?; + + println!( + "Completed in {elapsed:.2?} (vs ~{:.0?} sequential)", + std::time::Duration::from_millis(10 * n_trials as u64) + ); + println!( + "Best: f({:.3}, {:.3}) = {:.6}\n", + best.get(&x).unwrap(), + best.get(&y).unwrap(), + best.value + ); + + Ok(()) +} + +// ============================================================================ +// Section 2: Journal Storage +// ============================================================================ + +/// Persists trials to a JSONL file so that a study can be resumed later. +/// Useful for long-running experiments or crash recovery. +fn journal_storage_demo() -> optimizer::Result<()> { + println!("=== Section 2: Journal Storage ===\n"); + + let path = std::env::temp_dir().join("optimizer_advanced_example.jsonl"); + + // Clean up from any previous run + let _ = std::fs::remove_file(&path); + + let x = FloatParam::new(-5.0, 5.0).name("x"); + + // --- First run: optimize 20 trials and persist to disk --- + { + let storage = JournalStorage::<f64>::new(&path); + let study: Study<f64> = Study::builder() + .minimize() + .sampler(TpeSampler::new()) + .storage(storage) + .build(); + + study.optimize(20, |trial| { + let xv = x.suggest(trial)?; + Ok::<_, optimizer::Error>(xv * xv) + })?; + + println!( + "First run: {} trials saved to {}", + study.n_trials(), + path.display() + ); + } + + // --- Second run: resume from the journal file --- + { + // JournalStorage::open loads existing trials from disk + let storage = JournalStorage::<f64>::open(&path)?; + let study: Study<f64> = Study::builder() + .minimize() + .sampler(TpeSampler::new()) + .storage(storage) + .build(); + + // The sampler sees the prior 20 trials, so it starts informed + let before = study.n_trials(); + study.optimize(10, |trial| { + let xv = x.suggest(trial)?; + Ok::<_, optimizer::Error>(xv * xv) + })?; + + let best = study.best_trial()?; + println!( + "Resumed: {} → {} trials, best f({:.4}) = {:.6}", + before, + study.n_trials(), + best.get(&x).unwrap(), + best.value + ); + } + + // Clean up the temporary file + let _ = std::fs::remove_file(&path); + + println!(); + Ok(()) +} + +// ============================================================================ +// Section 3: Ask-and-Tell Interface +// ============================================================================ + +/// Decouples trial creation from evaluation. Useful when: +/// - Evaluations happen outside the optimizer (e.g., in a separate process) +/// - You want to batch evaluations before reporting results +/// - You need custom scheduling logic +fn ask_and_tell_demo() -> optimizer::Result<()> { + println!("=== Section 3: Ask-and-Tell Interface ===\n"); + + let study: Study<f64> = Study::new(Direction::Minimize); + + let x = FloatParam::new(-5.0, 5.0).name("x"); + let y = FloatParam::new(-5.0, 5.0).name("y"); + + // Ask for a batch of trials, evaluate externally, then tell results + for batch in 0..3 { + let batch_size = 5; + let mut trials = Vec::with_capacity(batch_size); + + // ask() creates trials with sampled parameters + for _ in 0..batch_size { + let mut trial = study.ask(); + let xv = x.suggest(&mut trial)?; + let yv = y.suggest(&mut trial)?; + + // Store values alongside the trial for later evaluation + trials.push((trial, xv, yv)); + } + + // Evaluate the batch (could be sent to workers, GPUs, etc.) + for (trial, xv, yv) in trials { + let value = xv * xv + yv * yv; + // tell() reports the result back to the study + study.tell(trial, Ok::<_, &str>(value)); + } + + println!( + "Batch {}: evaluated {} trials (total: {})", + batch + 1, + batch_size, + study.n_trials() + ); + } + + let best = study.best_trial()?; + println!( + "Best: f({:.3}, {:.3}) = {:.6}\n", + best.get(&x).unwrap(), + best.get(&y).unwrap(), + best.value + ); + + Ok(()) +} + +// ============================================================================ +// Section 4: Multi-Objective Optimization +// ============================================================================ + +/// Optimizes two competing objectives simultaneously. +/// Returns the Pareto front — the set of solutions where no objective can +/// be improved without worsening the other. +fn multi_objective_demo() -> optimizer::Result<()> { + println!("=== Section 4: Multi-Objective Optimization ===\n"); + + // Two objectives, both minimized + let study = MultiObjectiveStudy::new(vec![Direction::Minimize, Direction::Minimize]); + + let x = FloatParam::new(0.0, 1.0).name("x"); + + // Classic bi-objective problem: f1(x) = x², f2(x) = (x - 1)² + // The Pareto front is the curve where improving f1 worsens f2 and vice versa. + study.optimize(50, |trial| { + let xv = x.suggest(trial)?; + let f1 = xv * xv; + let f2 = (xv - 1.0) * (xv - 1.0); + Ok::<_, optimizer::Error>(vec![f1, f2]) + })?; + + let front = study.pareto_front(); + println!( + "Ran {} trials, Pareto front has {} solutions:", + study.n_trials(), + front.len() + ); + + // Show a few Pareto-optimal trade-offs + let mut sorted_front = front.clone(); + sorted_front.sort_by(|a, b| a.values[0].partial_cmp(&b.values[0]).unwrap()); + + for (i, trial) in sorted_front.iter().take(5).enumerate() { + println!( + " {}: x={:.3}, f1={:.4}, f2={:.4}", + i + 1, + trial.get(&x).unwrap(), + trial.values[0], + trial.values[1] + ); + } + if sorted_front.len() > 5 { + println!(" ... and {} more", sorted_front.len() - 5); + } + + println!(); + Ok(()) +} + +// ============================================================================ +// Main +// ============================================================================ + +#[tokio::main] +async fn main() -> optimizer::Result<()> { + async_parallel_optimization().await?; + journal_storage_demo()?; + ask_and_tell_demo()?; + multi_objective_demo()?; + + println!("All sections completed successfully!"); + Ok(()) +} diff --git a/examples/async_api_optimization.rs b/examples/async_api_optimization.rs deleted file mode 100644 index 7b41c4f..0000000 --- a/examples/async_api_optimization.rs +++ /dev/null @@ -1,368 +0,0 @@ -//! Async API Parameter Optimization Example -//! -//! This example shows how to use async/parallel optimization to tune -//! configuration parameters for a web service. Each evaluation simulates -//! an async operation (like deploying and load-testing a service). -//! -//! # Key Concepts Demonstrated -//! -//! - Async optimization with `optimize_parallel` -//! - Running multiple trials concurrently for faster optimization -//! - Boolean and categorical parameter types -//! - Measuring speedup from parallelism -//! -//! # When to Use Async Optimization -//! -//! Use async/parallel optimization when your objective function involves: -//! - Network requests (API calls, database queries) -//! - File I/O operations -//! - External service calls -//! - Any operation where you're waiting for I/O rather than computing -//! -//! With parallelism, you can evaluate multiple configurations simultaneously, -//! significantly reducing total optimization time. -//! -//! Run with: `cargo run --example async_api_optimization --features async` - -use std::time::{Duration, Instant}; - -use optimizer::prelude::*; - -// ============================================================================ -// Configuration: Service parameters we want to tune -// ============================================================================ - -/// Configuration for a web service. -/// -/// In a real application, these parameters would control: -/// - Memory allocation (cache sizes) -/// - Connection management (pool sizes, timeouts) -/// - Request handling (batching, compression) -/// - Protocol options (HTTP version, load balancing) -struct ServiceConfig { - cache_size_mb: i64, - connection_pool_size: i64, - request_timeout_ms: i64, - retry_count: i64, - batch_size: i64, - compression_level: i64, - use_http2: bool, - load_balancing: String, -} - -// ============================================================================ -// Objective Function: Evaluate a service configuration -// ============================================================================ - -/// Simulates deploying and load-testing a service configuration. -/// -/// In a real scenario, this function might: -/// 1. Deploy the configuration to a staging environment -/// 2. Run load tests against the service -/// 3. Collect metrics (latency, throughput, error rate) -/// 4. Return a composite score -/// -/// The async sleep simulates the I/O time of these operations. -/// This is where parallel execution helps - while one trial is waiting -/// for I/O, other trials can run. -#[allow(clippy::too_many_arguments)] -async fn evaluate_service(config: &ServiceConfig) -> f64 { - // Simulate async I/O (deployment, load testing, metric collection) - tokio::time::sleep(Duration::from_millis(50)).await; - - // Calculate a score based on how close we are to optimal values - // Lower score = better configuration - let mut score = 0.0; - - // Cache size: too small = cache misses, too large = wasted memory - // Optimal around 512MB - let cache_optimal = 512.0; - score += ((config.cache_size_mb as f64 - cache_optimal) / 256.0).powi(2); - - // Connection pool: too small = contention, too large = resource waste - // Optimal around 100 - let pool_optimal = 100.0; - score += ((config.connection_pool_size as f64 - pool_optimal) / 50.0).powi(2); - - // Timeout: too short = false failures, too long = slow recovery - // Optimal around 5000ms - let timeout_optimal = 5000.0; - score += ((config.request_timeout_ms as f64 - timeout_optimal) / 2000.0).powi(2); - - // Retries: too few = fragile, too many = amplifies failures - // Optimal around 3 - let retry_optimal = 3.0; - score += ((config.retry_count as f64 - retry_optimal) / 2.0).powi(2); - - // Batch size: trade-off between latency and throughput - // Optimal around 64 - let batch_optimal = 64.0; - score += ((config.batch_size as f64 - batch_optimal) / 32.0).powi(2); - - // Compression level: trade-off between CPU and bandwidth - // Optimal around 6 - let compression_optimal = 6.0; - score += ((config.compression_level as f64 - compression_optimal) / 3.0).powi(2); - - // HTTP/2 is generally better for our use case - if !config.use_http2 { - score += 0.5; - } - - // Load balancing strategy affects performance - score += match config.load_balancing.as_str() { - "round_robin" => 0.0, // Best for our use case - "least_connections" => 0.1, // Good alternative - "ip_hash" => 0.2, // OK for session affinity - "random" => 0.3, // Not ideal - _ => 1.0, - }; - - // Add noise to simulate real-world variability - let noise = (config.cache_size_mb as f64 * 0.1).sin() * 0.05; - - score + noise -} - -/// The async objective function for each trial. -/// -/// For async optimization, the objective function must: -/// 1. Take ownership of the Trial (not a mutable reference) -/// 2. Return a Future -/// 3. Return both the Trial and the result value as a tuple -/// -/// This ownership pattern allows the trial to be used across await points. -#[allow(clippy::too_many_arguments)] -async fn objective( - mut trial: Trial, - cache_size_mb_param: &IntParam, - connection_pool_size_param: &IntParam, - request_timeout_ms_param: &IntParam, - retry_count_param: &IntParam, - batch_size_param: &IntParam, - compression_level_param: &IntParam, - use_http2_param: &BoolParam, - load_balancing_param: &CategoricalParam<&str>, -) -> optimizer::Result<(Trial, f64)> { - // Sample configuration parameters using parameter definitions - let cache_size_mb = cache_size_mb_param.suggest(&mut trial)?; - let connection_pool_size = connection_pool_size_param.suggest(&mut trial)?; - let request_timeout_ms = request_timeout_ms_param.suggest(&mut trial)?; - let retry_count = retry_count_param.suggest(&mut trial)?; - let batch_size = batch_size_param.suggest(&mut trial)?; - let compression_level = compression_level_param.suggest(&mut trial)?; - let use_http2 = use_http2_param.suggest(&mut trial)?; - let load_balancing = load_balancing_param.suggest(&mut trial)?; - - // Build configuration - let config = ServiceConfig { - cache_size_mb, - connection_pool_size, - request_timeout_ms, - retry_count, - batch_size, - compression_level, - use_http2, - load_balancing: load_balancing.to_string(), - }; - - // Evaluate (this is the async part) - let score = evaluate_service(&config).await; - - // Return both the trial and the score - Ok((trial, score)) -} - -// ============================================================================ -// Helper Functions -// ============================================================================ - -/// Prints the results of the optimization. -fn print_results(study: &Study<f64>, elapsed: Duration, n_trials: usize) { - println!("\n{}", "=".repeat(60)); - println!("\nOptimization completed!"); - println!("Total trials: {}", study.n_trials()); - println!("Time elapsed: {elapsed:.2?}"); - - // Calculate speedup from parallelism - // Each trial takes ~50ms, so sequential would take n_trials * 50ms - let sequential_time = n_trials as f64 * 0.050; - let actual_time = elapsed.as_secs_f64(); - println!( - "Effective parallelism: {:.1}x speedup", - sequential_time / actual_time - ); -} - -/// Prints the best configuration found. -#[allow(clippy::too_many_arguments)] -fn print_best_config( - study: &Study<f64>, - cache_size_mb_param: &IntParam, - connection_pool_size_param: &IntParam, - request_timeout_ms_param: &IntParam, - retry_count_param: &IntParam, - batch_size_param: &IntParam, - compression_level_param: &IntParam, - use_http2_param: &BoolParam, - load_balancing_param: &CategoricalParam<&str>, -) -> optimizer::Result<()> { - let best = study.best_trial()?; - - println!("\nBest configuration found:"); - println!(" Score: {:.6}", best.value); - println!("\n Parameters:"); - println!( - " cache_size_mb: {}", - best.get(cache_size_mb_param).unwrap() - ); - println!( - " connection_pool_size: {}", - best.get(connection_pool_size_param).unwrap() - ); - println!( - " request_timeout_ms: {}", - best.get(request_timeout_ms_param).unwrap() - ); - println!(" retry_count: {}", best.get(retry_count_param).unwrap()); - println!(" batch_size: {}", best.get(batch_size_param).unwrap()); - println!( - " compression_level: {}", - best.get(compression_level_param).unwrap() - ); - println!(" use_http2: {}", best.get(use_http2_param).unwrap()); - println!( - " load_balancing: {}", - best.get(load_balancing_param).unwrap() - ); - - Ok(()) -} - -/// Prints the top N trials. -fn print_top_trials(study: &Study<f64>, n: usize) { - println!("\nTop {n} trials:"); - - let mut trials = study.trials(); - trials.sort_by(|a, b| a.value.partial_cmp(&b.value).unwrap()); - - for (i, trial) in trials.iter().take(n).enumerate() { - println!( - " {}. Trial #{}: score = {:.6}", - i + 1, - trial.id, - trial.value - ); - } -} - -// ============================================================================ -// Main: Set up and run the async optimization -// ============================================================================ - -#[tokio::main] -async fn main() -> optimizer::Result<()> { - println!("=== Async API Parameter Optimization Example ===\n"); - - // Step 1: Create a TPE sampler - let sampler = TpeSampler::builder() - .n_startup_trials(8) - .gamma(0.2) - .seed(123) - .build() - .expect("Failed to build TPE sampler"); - - // Step 2: Create a study to minimize the score - let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler); - - // Step 3: Define parameter search spaces - let cache_size_mb_param = IntParam::new(64, 1024).name("cache_size_mb").step(64); - let connection_pool_size_param = IntParam::new(10, 200).name("connection_pool_size").step(10); - let request_timeout_ms_param = IntParam::new(1000, 10000) - .name("request_timeout_ms") - .step(500); - let retry_count_param = IntParam::new(0, 5).name("retry_count"); - let batch_size_param = IntParam::new(1, 256).name("batch_size").log_scale(); - let compression_level_param = IntParam::new(0, 9).name("compression_level"); - let use_http2_param = BoolParam::new().name("use_http2"); - let load_balancing_param = CategoricalParam::new(vec![ - "round_robin", - "least_connections", - "random", - "ip_hash", - ]) - .name("load_balancing"); - - // Clone params for use after the closure moves them - let cache_size_mb_p = cache_size_mb_param.clone(); - let connection_pool_size_p = connection_pool_size_param.clone(); - let request_timeout_ms_p = request_timeout_ms_param.clone(); - let retry_count_p = retry_count_param.clone(); - let batch_size_p = batch_size_param.clone(); - let compression_level_p = compression_level_param.clone(); - let use_http2_p = use_http2_param.clone(); - let load_balancing_p = load_balancing_param.clone(); - - // Step 4: Configure optimization - let n_trials = 40; - let concurrency = 4; // Run 4 trials in parallel - - println!("Starting parallel optimization with {concurrency} concurrent evaluations...\n"); - - let start = Instant::now(); - - // Step 5: Run parallel async optimization - // - // optimize_parallel: - // - Runs up to `concurrency` trials simultaneously - // - Each trial calls the objective function - // - Uses a semaphore to limit concurrent evaluations - // - Collects results as trials complete - // - // The sampler gets access to trial history for informed sampling. - study - .optimize_parallel(n_trials, concurrency, move |trial| { - let cache_size_mb_param = cache_size_mb_param.clone(); - let connection_pool_size_param = connection_pool_size_param.clone(); - let request_timeout_ms_param = request_timeout_ms_param.clone(); - let retry_count_param = retry_count_param.clone(); - let batch_size_param = batch_size_param.clone(); - let compression_level_param = compression_level_param.clone(); - let use_http2_param = use_http2_param.clone(); - let load_balancing_param = load_balancing_param.clone(); - async move { - objective( - trial, - &cache_size_mb_param, - &connection_pool_size_param, - &request_timeout_ms_param, - &retry_count_param, - &batch_size_param, - &compression_level_param, - &use_http2_param, - &load_balancing_param, - ) - .await - } - }) - .await?; - - let elapsed = start.elapsed(); - - // Step 5: Print results - print_results(&study, elapsed, n_trials); - print_best_config( - &study, - &cache_size_mb_p, - &connection_pool_size_p, - &request_timeout_ms_p, - &retry_count_p, - &batch_size_p, - &compression_level_p, - &use_http2_p, - &load_balancing_p, - )?; - print_top_trials(&study, 5); - - Ok(()) -} diff --git a/examples/basic_optimization.rs b/examples/basic_optimization.rs new file mode 100644 index 0000000..097853a --- /dev/null +++ b/examples/basic_optimization.rs @@ -0,0 +1,32 @@ +//! Basic optimization example — the "hello world" of the optimizer crate. +//! +//! Minimizes a simple quadratic function f(x) = (x - 3)² using the default +//! random sampler. No feature flags are required. +//! +//! Run with: `cargo run --example basic_optimization` + +use optimizer::prelude::*; + +fn main() { + // Create a study that minimizes the objective function. + // The default sampler is random; for smarter sampling, pass a TpeSampler. + let study: Study<f64> = Study::new(Direction::Minimize); + + // Search for x in [-10, 10]. The optimizer will suggest values from this range. + let x = FloatParam::new(-10.0, 10.0).name("x"); + + // Run 50 trials, each evaluating f(x) = (x - 3)² + study + .optimize(50, |trial| { + let x_val = x.suggest(trial)?; + let value = (x_val - 3.0).powi(2); + Ok::<_, Error>(value) + }) + .unwrap(); + + // Retrieve and display the best result + let best = study.best_trial().unwrap(); + println!("Best trial #{}", best.id); + println!(" x = {:.4}", best.get(&x).unwrap()); + println!(" f(x) = {:.4}", best.value); +} diff --git a/examples/benchmark_convergence.rs b/examples/benchmark_convergence.rs deleted file mode 100644 index 9f478a7..0000000 --- a/examples/benchmark_convergence.rs +++ /dev/null @@ -1,124 +0,0 @@ -use std::ops::ControlFlow; -use std::time::Instant; - -use optimizer::parameter::Parameter; -use optimizer::sampler::random::RandomSampler; -use optimizer::sampler::tpe::TpeSampler; -use optimizer::{FloatParam, Study}; - -/// Standard optimization test functions. -mod functions { - pub fn sphere(x: &[f64]) -> f64 { - x.iter().map(|xi| xi * xi).sum() - } - - pub fn rosenbrock(x: &[f64]) -> f64 { - x.windows(2) - .map(|w| 100.0 * (w[1] - w[0] * w[0]).powi(2) + (1.0 - w[0]).powi(2)) - .sum() - } - - pub fn rastrigin(x: &[f64]) -> f64 { - let n = x.len() as f64; - 10.0 * n - + x.iter() - .map(|xi| xi * xi - 10.0 * (2.0 * std::f64::consts::PI * xi).cos()) - .sum::<f64>() - } -} - -fn run_convergence( - name: &str, - sampler_name: &str, - study: Study<f64>, - params: &[FloatParam], - objective: fn(&[f64]) -> f64, - n_trials: usize, -) { - let start = Instant::now(); - - study - .optimize_with_callback( - n_trials, - |trial| { - let x: Vec<f64> = params - .iter() - .map(|p| p.suggest(trial)) - .collect::<Result<_, _>>() - .unwrap(); - Ok::<_, optimizer::Error>(objective(&x)) - }, - |study, _trial| { - let elapsed = start.elapsed().as_millis(); - let best = study.best_value().unwrap(); - let n = study.n_trials(); - println!("{n},{best},{elapsed},{sampler_name},{name}"); - ControlFlow::Continue(()) - }, - ) - .unwrap(); -} - -fn main() { - println!("trial,best_value,elapsed_ms,sampler,function"); - - let dims = 5; - let params: Vec<FloatParam> = (0..dims) - .map(|i| FloatParam::new(-5.0, 5.0).name(format!("x{i}"))) - .collect(); - let n_trials = 200; - - // Sphere: Random vs TPE - run_convergence( - "sphere_5d", - "random", - Study::minimize(RandomSampler::with_seed(1)), - ¶ms, - functions::sphere, - n_trials, - ); - run_convergence( - "sphere_5d", - "tpe", - Study::minimize(TpeSampler::builder().seed(1).build().unwrap()), - ¶ms, - functions::sphere, - n_trials, - ); - - // Rosenbrock: Random vs TPE - run_convergence( - "rosenbrock_5d", - "random", - Study::minimize(RandomSampler::with_seed(2)), - ¶ms, - functions::rosenbrock, - n_trials, - ); - run_convergence( - "rosenbrock_5d", - "tpe", - Study::minimize(TpeSampler::builder().seed(2).build().unwrap()), - ¶ms, - functions::rosenbrock, - n_trials, - ); - - // Rastrigin: Random vs TPE - run_convergence( - "rastrigin_5d", - "random", - Study::minimize(RandomSampler::with_seed(3)), - ¶ms, - functions::rastrigin, - n_trials, - ); - run_convergence( - "rastrigin_5d", - "tpe", - Study::minimize(TpeSampler::builder().seed(3).build().unwrap()), - ¶ms, - functions::rastrigin, - n_trials, - ); -} diff --git a/examples/ml_hyperparameter_tuning.rs b/examples/ml_hyperparameter_tuning.rs deleted file mode 100644 index d73973d..0000000 --- a/examples/ml_hyperparameter_tuning.rs +++ /dev/null @@ -1,275 +0,0 @@ -//! Machine Learning Hyperparameter Tuning Example -//! -//! This example shows how to use the optimizer library to find the best -//! hyperparameters for a machine learning model. We simulate a gradient -//! boosting model (like XGBoost or LightGBM) and search for optimal settings. -//! -//! # Key Concepts Demonstrated -//! -//! - Creating a Study with a TPE (Tree-Parzen Estimator) sampler -//! - Defining an objective function that the optimizer will minimize -//! - Using different parameter types: floats, integers, log-scale, stepped -//! - Using callbacks to monitor progress and implement early stopping -//! -//! # How It Works -//! -//! 1. Create a `Study` - this manages the optimization process -//! 2. Define an objective function that takes a `Trial` and returns a score -//! 3. Inside the objective, use `trial.suggest_*()` to sample parameters -//! 4. The optimizer runs many trials, learning which parameter regions work best -//! 5. After optimization, retrieve the best parameters found -//! -//! Run with: `cargo run --example ml_hyperparameter_tuning` - -use std::ops::ControlFlow; - -use optimizer::prelude::*; - -// ============================================================================ -// Configuration: Hyperparameters we want to tune -// ============================================================================ - -/// Holds all the hyperparameters for our model. -/// -/// In a real application, you would pass these to your ML framework -/// (e.g., XGBoost, LightGBM, scikit-learn). -struct ModelConfig { - learning_rate: f64, - max_depth: i64, - n_estimators: i64, - subsample: f64, - colsample_bytree: f64, - min_child_weight: i64, - reg_alpha: f64, - reg_lambda: f64, -} - -// ============================================================================ -// Objective Function: What we want to optimize -// ============================================================================ - -/// Simulates training a model and returns the validation loss. -/// -/// In a real scenario, this function would: -/// 1. Create a model with the given hyperparameters -/// 2. Train it on your training data -/// 3. Evaluate it on validation data -/// 4. Return the validation metric (e.g., RMSE, log loss, accuracy) -/// -/// The optimizer will try to MINIMIZE this value (we set Direction::Minimize). -#[allow(clippy::too_many_arguments)] -fn evaluate_model(config: &ModelConfig) -> f64 { - // Simulated optimal hyperparameters: - // learning_rate ~ 0.05, max_depth ~ 6, n_estimators ~ 200 - // subsample ~ 0.8, colsample_bytree ~ 0.8, min_child_weight ~ 3 - // reg_alpha ~ 0.1, reg_lambda ~ 1.0 - - let mut loss = 0.15; // Base loss - - // Each term penalizes deviation from the optimal value - loss += (config.learning_rate - 0.05).powi(2) * 100.0; - loss += ((config.max_depth - 6) as f64).powi(2) * 0.01; - loss += ((config.n_estimators - 200) as f64).powi(2) * 0.00001; - loss += (config.subsample - 0.8).powi(2) * 10.0; - loss += (config.colsample_bytree - 0.8).powi(2) * 10.0; - loss += ((config.min_child_weight - 3) as f64).powi(2) * 0.05; - loss += (config.reg_alpha - 0.1).powi(2) * 5.0; - loss += (config.reg_lambda - 1.0).powi(2) * 2.0; - - // Add some noise to simulate real-world variability - let noise = (config.learning_rate * 1000.0).sin() * 0.01; - - loss + noise -} - -/// The objective function that the optimizer calls for each trial. -/// -/// This function: -/// 1. Uses parameter definitions passed as arguments -/// 2. Builds a model configuration from the suggested values -/// 3. Evaluates the model and returns the loss -/// -/// The optimizer learns from the results to suggest better parameters -/// in future trials. -#[allow(clippy::too_many_arguments)] -fn objective( - trial: &mut Trial, - learning_rate_param: &FloatParam, - max_depth_param: &IntParam, - n_estimators_param: &IntParam, - subsample_param: &FloatParam, - colsample_bytree_param: &FloatParam, - min_child_weight_param: &IntParam, - reg_alpha_param: &FloatParam, - reg_lambda_param: &FloatParam, -) -> optimizer::Result<f64> { - let learning_rate = learning_rate_param.suggest(trial)?; - let max_depth = max_depth_param.suggest(trial)?; - let n_estimators = n_estimators_param.suggest(trial)?; - let subsample = subsample_param.suggest(trial)?; - let colsample_bytree = colsample_bytree_param.suggest(trial)?; - let min_child_weight = min_child_weight_param.suggest(trial)?; - let reg_alpha = reg_alpha_param.suggest(trial)?; - let reg_lambda = reg_lambda_param.suggest(trial)?; - - // Build configuration and evaluate - let config = ModelConfig { - learning_rate, - max_depth, - n_estimators, - subsample, - colsample_bytree, - min_child_weight, - reg_alpha, - reg_lambda, - }; - - let loss = evaluate_model(&config); - - Ok(loss) -} - -// ============================================================================ -// Callback Function: Monitor progress and implement early stopping -// ============================================================================ - -/// Called after each successful trial completes. -/// -/// Use callbacks to: -/// - Log progress to console or file -/// - Save checkpoints -/// - Implement early stopping when a good solution is found -/// - Track metrics over time -/// -/// Return `ControlFlow::Continue(())` to keep optimizing. -/// Return `ControlFlow::Break(())` to stop early. -fn on_trial_complete(study: &Study<f64>, trial: &CompletedTrial<f64>) -> ControlFlow<()> { - // Print trial number and objective value - print!("{:>5} ", study.n_trials()); - for value in trial.params.values() { - print!("{value:>12} "); - } - println!("{:>12.6}", trial.value); - - // Early stopping: if we find an excellent solution, stop early - if trial.value < 0.16 { - println!("\nEarly stopping: found excellent solution!"); - return ControlFlow::Break(()); - } - - ControlFlow::Continue(()) -} - -// ============================================================================ -// Main: Set up and run the optimization -// ============================================================================ - -fn main() -> optimizer::Result<()> { - println!("=== ML Hyperparameter Tuning Example ===\n"); - - // Step 1: Create a sampler - // - // TPE (Tree-Parzen Estimator) is a Bayesian optimization algorithm. - // It learns from previous trials to suggest better parameters. - // - n_startup_trials: Number of random trials before TPE kicks in - // - gamma: What fraction of trials are considered "good" (lower = more selective) - // - seed: For reproducibility - let sampler = TpeSampler::builder() - .n_startup_trials(10) - .gamma(0.25) - .seed(42) - .build() - .expect("Failed to build TPE sampler"); - - // Step 2: Create a study - // - // The study manages the optimization process. We want to MINIMIZE - // the loss (lower is better). Use Direction::Maximize for metrics - // where higher is better (like accuracy). - let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler); - - // Print header - println!("Starting hyperparameter optimization...\n"); - println!( - "{:>5} {:>12} (parameters...) {:>12}", - "Trial", "Params", "Loss" - ); - println!("{}", "-".repeat(60)); - - // Step 3: Define parameter search spaces - let learning_rate_param = FloatParam::new(0.001, 0.3) - .name("learning_rate") - .log_scale(); - let max_depth_param = IntParam::new(3, 12).name("max_depth"); - let n_estimators_param = IntParam::new(50, 500).name("n_estimators").step(50); - let subsample_param = FloatParam::new(0.5, 1.0).name("subsample"); - let colsample_bytree_param = FloatParam::new(0.5, 1.0).name("colsample_bytree"); - let min_child_weight_param = IntParam::new(1, 10).name("min_child_weight"); - let reg_alpha_param = FloatParam::new(1e-3, 10.0).name("reg_alpha").log_scale(); - let reg_lambda_param = FloatParam::new(1e-3, 10.0).name("reg_lambda").log_scale(); - - // Step 4: Run optimization - // - // optimize_with_callback runs the objective function for up to - // n_trials iterations. After each trial, it calls the callback. - // The sampler gets access to trial history for informed sampling. - let n_trials = 50; - - study.optimize_with_callback( - n_trials, - |trial| { - objective( - trial, - &learning_rate_param, - &max_depth_param, - &n_estimators_param, - &subsample_param, - &colsample_bytree_param, - &min_child_weight_param, - ®_alpha_param, - ®_lambda_param, - ) - }, - on_trial_complete, - )?; - - // Step 4: Get the best result - println!("\n{}", "=".repeat(110)); - println!("\nOptimization completed!"); - println!("Total trials: {}", study.n_trials()); - - let best = study.best_trial()?; - println!("\nBest trial:"); - println!(" Loss: {:.6}", best.value); - println!(" Parameters:"); - println!( - " learning_rate: {:.6}", - best.get(&learning_rate_param).unwrap() - ); - println!(" max_depth: {}", best.get(&max_depth_param).unwrap()); - println!( - " n_estimators: {}", - best.get(&n_estimators_param).unwrap() - ); - println!(" subsample: {:.6}", best.get(&subsample_param).unwrap()); - println!( - " colsample_bytree: {:.6}", - best.get(&colsample_bytree_param).unwrap() - ); - println!( - " min_child_weight: {}", - best.get(&min_child_weight_param).unwrap() - ); - println!(" reg_alpha: {:.6}", best.get(®_alpha_param).unwrap()); - println!( - " reg_lambda: {:.6}", - best.get(®_lambda_param).unwrap() - ); - - // Step 5: Use the best parameters (in a real app) - // - // Now you would take best.params and use them to train your final model - // on the full dataset. - - Ok(()) -} diff --git a/examples/parameter_api.rs b/examples/parameter_api.rs deleted file mode 100644 index 119fcf4..0000000 --- a/examples/parameter_api.rs +++ /dev/null @@ -1,56 +0,0 @@ -use optimizer::prelude::*; -use optimizer_derive::Categorical; - -#[derive(Clone, Debug, Categorical)] -enum Activation { - Relu, - Sigmoid, - Tanh, -} - -fn main() { - let study: Study<f64> = Study::new(Direction::Minimize); - - // Define parameters outside the objective function - let lr_param = FloatParam::new(1e-5, 1e-1).name("lr").log_scale(); - let n_layers_param = IntParam::new(1, 5).name("n_layers"); - let units_param = IntParam::new(32, 512).name("units").step(32); - let optimizer_param = CategoricalParam::new(vec!["sgd", "adam", "rmsprop"]).name("optimizer"); - let activation_param = EnumParam::<Activation>::new().name("activation"); - let batch_size_param = IntParam::new(16, 256).name("batch_size").log_scale(); - let use_dropout_param = BoolParam::new().name("use_dropout"); - - study - .optimize(20, |trial| { - let lr = lr_param.suggest(trial)?; - let n_layers = n_layers_param.suggest(trial)?; - let units = units_param.suggest(trial)?; - let optimizer = optimizer_param.suggest(trial)?; - let use_dropout = use_dropout_param.suggest(trial)?; - let activation = activation_param.suggest(trial)?; - let batch_size = batch_size_param.suggest(trial)?; - - // Simulate a loss function - let loss = lr * (n_layers as f64) + (units as f64) * 0.001 - - if use_dropout { 0.1 } else { 0.0 }; - - println!( - "Trial {}: lr={lr:.6}, layers={n_layers}, units={units}, opt={optimizer}, \ - dropout={use_dropout}, activation={activation:?}, batch={batch_size} -> loss={loss:.4}", - trial.id() - ); - - Ok::<_, Error>(loss) - }) - .unwrap(); - - let best = study.best_trial().unwrap(); - println!("\nBest trial: value={:.4}", best.value); - println!(" lr: {:.6}", best.get(&lr_param).unwrap()); - println!(" n_layers: {}", best.get(&n_layers_param).unwrap()); - println!(" units: {}", best.get(&units_param).unwrap()); - println!(" optimizer: {}", best.get(&optimizer_param).unwrap()); - println!(" activation: {:?}", best.get(&activation_param).unwrap()); - println!(" batch_size: {}", best.get(&batch_size_param).unwrap()); - println!(" use_dropout: {}", best.get(&use_dropout_param).unwrap()); -} diff --git a/examples/parameter_types.rs b/examples/parameter_types.rs new file mode 100644 index 0000000..c509c5b --- /dev/null +++ b/examples/parameter_types.rs @@ -0,0 +1,73 @@ +//! Parameter types example — demonstrates all five parameter types and the derive feature. +//! +//! Shows `FloatParam`, `IntParam`, `CategoricalParam`, `BoolParam`, and `EnumParam` +//! with `.name()` labels, `#[derive(Categorical)]` for enums, and typed access +//! to results via `CompletedTrial::get()`. +//! +//! Run with: `cargo run --example parameter_types --features derive` + +use optimizer::prelude::*; +use optimizer_derive::Categorical; + +/// Activation functions — `#[derive(Categorical)]` auto-generates the +/// `Categorical` trait, mapping each variant to a sequential index. +#[derive(Clone, Debug, Categorical)] +enum Activation { + Relu, + Sigmoid, + Tanh, + Gelu, +} + +fn main() { + let study: Study<f64> = Study::new(Direction::Minimize); + + // --- Define one of each parameter type, each with a human-readable .name() --- + + // Float: learning rate on a log scale (common for ML hyperparameters) + let lr = FloatParam::new(1e-5, 1e-1).log_scale().name("lr"); + + // Int: number of hidden layers (stepped by 1, the default) + let n_layers = IntParam::new(1, 5).name("n_layers"); + + // Categorical: optimizer algorithm chosen from a list of strings + let optimizer = CategoricalParam::new(vec!["sgd", "adam", "rmsprop"]).name("optimizer"); + + // Bool: whether to apply dropout + let use_dropout = BoolParam::new().name("use_dropout"); + + // Enum: activation function — uses #[derive(Categorical)] above + let activation = EnumParam::<Activation>::new().name("activation"); + + // --- Run the optimization --- + study + .optimize(30, |trial| { + let lr_val = lr.suggest(trial)?; + let layers = n_layers.suggest(trial)?; + let opt = optimizer.suggest(trial)?; + let dropout = use_dropout.suggest(trial)?; + let act = activation.suggest(trial)?; + + // Simulated loss that depends on all parameters + let loss = lr_val * f64::from(layers as i32) + + if opt == "adam" { -0.05 } else { 0.0 } + + if dropout { -0.02 } else { 0.0 } + + match act { + Activation::Gelu => -0.03, + Activation::Relu => -0.01, + _ => 0.0, + }; + + Ok::<_, Error>(loss) + }) + .unwrap(); + + // --- Retrieve best trial and read back each parameter with typed .get() --- + let best = study.best_trial().unwrap(); + println!("Best trial #{} — loss = {:.6}", best.id, best.value); + println!(" lr = {:.6}", best.get(&lr).unwrap()); + println!(" n_layers = {}", best.get(&n_layers).unwrap()); + println!(" optimizer = {}", best.get(&optimizer).unwrap()); + println!(" use_dropout = {}", best.get(&use_dropout).unwrap()); + println!(" activation = {:?}", best.get(&activation).unwrap()); +} diff --git a/examples/pruning_and_callbacks.rs b/examples/pruning_and_callbacks.rs new file mode 100644 index 0000000..207e55b --- /dev/null +++ b/examples/pruning_and_callbacks.rs @@ -0,0 +1,133 @@ +//! Pruning and early-stopping example — demonstrates trial pruning with `MedianPruner` +//! and early stopping via `optimize_with_callback`. +//! +//! Simulates a training loop where each trial trains for multiple "epochs". The pruner +//! stops unpromising trials early, and a callback halts the entire study once a target +//! loss is reached. +//! +//! Run with: `cargo run --example pruning_and_callbacks` + +use std::ops::ControlFlow; + +use optimizer::TrialState; +use optimizer::prelude::*; + +fn main() -> optimizer::Result<()> { + let n_trials: usize = 30; + let n_epochs: u64 = 20; + let target_loss = 0.15; + + // Build a study with a seeded random sampler and MedianPruner. + // MedianPruner compares each trial's intermediate value against the median of + // completed trials at the same step — trials performing below median are pruned. + let study: Study<f64> = Study::builder() + .minimize() + .sampler(RandomSampler::with_seed(42)) + .pruner( + MedianPruner::new(Direction::Minimize) + .n_warmup_steps(3) // let every trial run at least 3 epochs before pruning + .n_min_trials(3), // need 3 completed trials before pruning kicks in + ) + .build(); + + let learning_rate = FloatParam::new(1e-4, 1.0).name("learning_rate"); + let momentum = FloatParam::new(0.0, 0.99).name("momentum"); + + // Use optimize_with_callback to get both pruning AND early stopping. + // The callback fires after each completed (or pruned) trial and can halt the study. + study.optimize_with_callback( + n_trials, + // --- Objective function: simulated training loop with pruning --- + |trial| { + let lr = learning_rate.suggest(trial)?; + let mom = momentum.suggest(trial)?; + + // Simulate training for n_epochs, reporting intermediate loss each epoch. + // Good hyperparameters (lr ≈ 0.01, momentum ≈ 0.8) converge to low loss; + // bad combos plateau high — giving the pruner something to cut. + let mut loss = 1.0; + for epoch in 0..n_epochs { + let lr_penalty = (lr.log10() - 0.01_f64.log10()).powi(2); // 0 at lr=0.01 + let mom_penalty = (mom - 0.8).powi(2); // 0 at momentum=0.8 + let base_loss = 0.02 + 0.05 * lr_penalty + 1.5 * mom_penalty; + let progress = (epoch as f64 + 1.0) / n_epochs as f64; + // Loss decays from 1.0 toward base_loss over epochs. + loss = base_loss + (1.0 - base_loss) * (-3.5 * progress).exp(); + + // Report the intermediate value so the pruner can evaluate this trial. + trial.report(epoch, loss); + + // Check whether the pruner recommends stopping this trial early. + if trial.should_prune() { + // Signal that this trial was pruned — the study records it as Pruned. + Err(TrialPruned)?; + } + } + + Ok::<_, Error>(loss) + }, + // --- Callback: early stopping when we hit the target --- + |study, completed_trial| { + let n_complete = study.n_trials(); + let n_pruned = study + .trials() + .iter() + .filter(|t| t.state == TrialState::Pruned) + .count(); + + match completed_trial.state { + TrialState::Pruned => { + println!( + " Trial {:>3} PRUNED at epoch {} (loss = {:.4}) \ + [{n_complete} done, {n_pruned} pruned]", + completed_trial.id, + completed_trial.intermediate_values.len(), + completed_trial + .intermediate_values + .last() + .map_or(f64::NAN, |v| v.1), + ); + } + TrialState::Complete => { + println!( + " Trial {:>3} complete: loss = {:.4} \ + [{n_complete} done, {n_pruned} pruned]", + completed_trial.id, completed_trial.value, + ); + } + _ => {} + } + + // Stop the entire study once we find a good enough result. + if completed_trial.state == TrialState::Complete && completed_trial.value < target_loss + { + println!("\n Early stopping: reached target loss {target_loss}!"); + return ControlFlow::Break(()); + } + + ControlFlow::Continue(()) + }, + )?; + + // --- Results --- + let best = study.best_trial().expect("at least one completed trial"); + let total = study.n_trials(); + let pruned = study + .trials() + .iter() + .filter(|t| t.state == TrialState::Pruned) + .count(); + + println!("\n--- Results ---"); + println!(" Total trials : {total}"); + println!(" Pruned : {pruned}"); + println!(" Completed : {}", total - pruned); + println!(" Best trial #{}: loss = {:.6}", best.id, best.value); + println!( + " learning_rate = {:.6}", + best.get(&learning_rate).unwrap() + ); + println!(" momentum = {:.4}", best.get(&momentum).unwrap()); + + Ok(()) +} diff --git a/examples/sampler_comparison.rs b/examples/sampler_comparison.rs new file mode 100644 index 0000000..b743e2a --- /dev/null +++ b/examples/sampler_comparison.rs @@ -0,0 +1,94 @@ +//! Sampler comparison example — benchmarks Random, TPE, and Grid samplers on the same problem. +//! +//! Runs the Sphere function f(x, y) = x² + y² with each sampler and compares the best +//! value found. This shows how sampler choice affects optimization quality. +//! +//! Run with: `cargo run --example sampler_comparison` + +use optimizer::prelude::*; + +/// Shared objective function: Sphere function with global minimum at (0, 0). +/// Simple enough to solve well, but 2-D so samplers have room to differ. +fn sphere(x: f64, y: f64) -> f64 { + x.powi(2) + y.powi(2) +} + +/// Run an optimization study and return the best value found. +fn run_study(study: Study<f64>, n_trials: usize) -> f64 { + // Use asymmetric ranges so the Grid sampler tracks each parameter independently. + let x = FloatParam::new(-5.0, 5.0).name("x"); + let y = FloatParam::new(-3.0, 3.0).name("y"); + + study + .optimize(n_trials, |trial| { + let x_val = x.suggest(trial)?; + let y_val = y.suggest(trial)?; + Ok::<_, Error>(sphere(x_val, y_val)) + }) + .unwrap(); + + let best = study.best_trial().unwrap(); + println!( + " Best trial #{:>3}: x = {:>7.4}, y = {:>7.4}, f(x,y) = {:.6}", + best.id, + best.get(&x).unwrap(), + best.get(&y).unwrap(), + best.value, + ); + best.value +} + +fn main() { + let n_trials: usize = 100; + println!("Comparing samplers on Sphere(x, y) = x² + y²"); + println!(" Search space: x ∈ [-5, 5], y ∈ [-3, 3]"); + println!(" Known minimum: f(0, 0) = 0"); + println!(" Trials per sampler: {n_trials}"); + println!(); + + // --- Random sampler (baseline) --- + // Pure random search: samples uniformly at random. Fast but not guided. + println!("1. Random sampler:"); + let random_best = run_study(Study::minimize(RandomSampler::with_seed(42)), n_trials); + + // --- TPE sampler (Bayesian) --- + // Tree-structured Parzen Estimator: builds a probabilistic model of good vs bad + // regions and focuses sampling where improvements are likely. + println!("\n2. TPE sampler (Bayesian):"); + let tpe = TpeSampler::builder() + .n_startup_trials(10) // random exploration for the first 10 trials + .n_ei_candidates(24) // candidates evaluated per Expected Improvement step + .gamma(0.25) // top 25% of trials define the "good" distribution + .seed(42) + .build() + .unwrap(); + let tpe_best = run_study(Study::minimize(tpe), n_trials); + + // --- Grid sampler (exhaustive) --- + // Evaluates evenly spaced grid points. Each parameter gets its own grid that + // is sampled in order, so n_points_per_param must be >= n_trials. + println!("\n3. Grid sampler (exhaustive):"); + let grid = GridSearchSampler::builder() + .n_points_per_param(n_trials) // one grid point per trial per parameter + .build(); + let grid_best = run_study(Study::minimize(grid), n_trials); + + // --- Summary --- + println!("\n--- Summary ---"); + println!(" Random : {random_best:.6}"); + println!(" TPE : {tpe_best:.6}"); + println!(" Grid : {grid_best:.6}"); + println!(); + + // Find the winner + let results = [ + ("Random", random_best), + ("TPE", tpe_best), + ("Grid", grid_best), + ]; + let (winner, _) = results + .iter() + .min_by(|a, b| a.1.partial_cmp(&b.1).unwrap()) + .unwrap(); + println!("Winner: {winner} (closest to known minimum of 0.0)"); +} diff --git a/examples/visualization_demo.rs b/examples/visualization_demo.rs deleted file mode 100644 index e2efd76..0000000 --- a/examples/visualization_demo.rs +++ /dev/null @@ -1,45 +0,0 @@ -use optimizer::prelude::*; - -fn main() { - // Multi-parameter optimization with TPE sampler. - let sampler = TpeSampler::builder().seed(42).build().unwrap(); - let mut study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler); - study.set_pruner(MedianPruner::new(Direction::Minimize)); - - let lr = FloatParam::new(1e-5, 1e-1) - .log_scale() - .name("learning_rate"); - let n_layers = IntParam::new(1, 5).name("n_layers"); - let dropout = FloatParam::new(0.0, 0.5).step(0.05).name("dropout"); - let batch_size = CategoricalParam::new(vec![16, 32, 64, 128]).name("batch_size"); - - study - .optimize(80, |trial| { - let lr_val = lr.suggest(trial)?; - let layers = n_layers.suggest(trial)?; - let drop = dropout.suggest(trial)?; - let bs = batch_size.suggest(trial)?; - - // Simulate training with intermediate reporting. - let mut loss = 1.0; - for epoch in 0..10 { - loss *= 0.7 + 0.3 * lr_val.ln().abs() / 12.0; - loss += drop * 0.05; - loss += (1.0 / bs as f64) * 0.1; - loss -= layers as f64 * 0.02; - trial.report(epoch, loss); - if trial.should_prune() { - return Err(TrialPruned.into()); - } - } - - Ok::<_, Error>(loss) - }) - .unwrap(); - - println!("{}", study.summary()); - - let path = "optimization_report.html"; - generate_html_report(&study, path).unwrap(); - println!("\nReport saved to {path}"); -} diff --git a/src/error.rs b/src/error.rs index 130534e..70b2d98 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,6 +1,21 @@ +//! Error types for the optimizer crate. +//! +//! All fallible operations in the crate return [`Result<T>`], which is an +//! alias for `core::result::Result<T, Error>`. The [`Error`] enum covers +//! parameter validation, sampling conflicts, pruning signals, and +//! feature-gated I/O errors. + +/// Errors returned by optimizer operations. +/// +/// Most variants are returned during parameter validation or trial +/// management. The [`TrialPruned`](Error::TrialPruned) variant has special +/// significance — it signals early stopping and is typically raised via +/// the [`TrialPruned`](super::TrialPruned) convenience type. #[derive(Debug, thiserror::Error)] pub enum Error { - /// Returned when the lower bound is greater than the upper bound. + /// The lower bound exceeds the upper bound in a + /// [`FloatParam`](crate::parameter::FloatParam) or + /// [`IntParam`](crate::parameter::IntParam). #[error("invalid bounds: low ({low}) must be less than or equal to high ({high})")] InvalidBounds { /// The lower bound value. @@ -9,19 +24,22 @@ pub enum Error { high: f64, }, - /// Returned when log scale is used with non-positive bounds. + /// Log-scale is enabled but the lower bound is not positive (float) or + /// is less than 1 (integer). #[error("invalid log bounds: low must be positive for log scale")] InvalidLogBounds, - /// Returned when step size is not positive. + /// The step size provided to a parameter is not positive. #[error("invalid step: step must be positive")] InvalidStep, - /// Returned when categorical choices are empty. + /// A [`CategoricalParam`](crate::parameter::CategoricalParam) was created + /// with an empty choices vector. #[error("categorical choices cannot be empty")] EmptyChoices, - /// Returned when a parameter is suggested with a different configuration. + /// The same [`ParamId`](crate::parameter::ParamId) was suggested twice + /// with a different distribution configuration. #[error("parameter conflict for '{name}': {reason}")] ParameterConflict { /// The name of the conflicting parameter. @@ -30,27 +48,29 @@ pub enum Error { reason: String, }, - /// Returned when requesting the best trial but no trials have completed. + /// [`Study::best_trial`](crate::Study::best_trial) or similar was called + /// before any trial completed successfully. #[error("no completed trials available")] NoCompletedTrials, - /// Returned when gamma is not in the valid range (0.0, 1.0). + /// The gamma value for TPE sampling is outside the open interval (0, 1). #[error("invalid gamma: {0} must be in (0.0, 1.0)")] InvalidGamma(f64), - /// Returned when bandwidth is not positive. + /// A KDE bandwidth value is not positive. #[error("invalid bandwidth: {0} must be positive")] InvalidBandwidth(f64), - /// Returned when KDE is created with empty samples. + /// A kernel density estimator was constructed with no samples. #[error("KDE requires at least one sample")] EmptySamples, - /// Returned when multivariate KDE samples have zero dimensions. + /// Multivariate KDE samples have zero dimensions. #[error("multivariate KDE samples must have at least one dimension")] ZeroDimensions, - /// Returned when multivariate KDE samples have inconsistent dimensions. + /// A sample in the multivariate KDE has a different number of dimensions + /// than the first sample. #[error( "dimension mismatch: expected {expected} dimensions but sample {sample_index} has {got}" )] @@ -63,7 +83,7 @@ pub enum Error { sample_index: usize, }, - /// Returned when bandwidth vector length doesn't match the number of dimensions. + /// The bandwidth vector length does not match the number of KDE dimensions. #[error("bandwidth dimension mismatch: expected {expected} bandwidths but got {got}")] BandwidthDimensionMismatch { /// The expected number of bandwidths. @@ -72,11 +92,14 @@ pub enum Error { got: usize, }, - /// Returned when a trial is pruned (stopped early by the objective function). + /// The objective signalled that this trial should be pruned (stopped + /// early). Typically raised via `Err(TrialPruned)?` inside the + /// objective closure. #[error("trial was pruned")] TrialPruned, - /// Returned when the objective returns the wrong number of values. + /// The multi-objective closure returned a different number of values + /// than the number of directions configured on the study. #[error("objective dimension mismatch: expected {expected} values, got {got}")] ObjectiveDimensionMismatch { /// The expected number of objective values. @@ -85,21 +108,24 @@ pub enum Error { got: usize, }, - /// Returned when an internal invariant is violated. + /// An internal invariant was violated. This indicates a bug in the + /// library rather than a user error. #[error("internal error: {0}")] Internal(&'static str), - /// Returned when an async task fails. + /// An async worker task failed. Only available with the `async` feature. #[cfg(feature = "async")] #[error("async task error: {0}")] TaskError(String), - /// Returned when a storage operation fails. + /// A storage I/O operation failed. Only available with the `journal` + /// feature. #[cfg(feature = "journal")] #[error("storage error: {0}")] Storage(String), } +/// A convenience alias for `core::result::Result<T, Error>`. pub type Result<T> = core::result::Result<T, Error>; /// Convenience type for signalling a pruned trial from an objective function. diff --git a/src/fanova.rs b/src/fanova.rs index 52882f6..91b6af0 100644 --- a/src/fanova.rs +++ b/src/fanova.rs @@ -1,26 +1,83 @@ //! fANOVA (functional ANOVA) parameter importance via random forest. //! -//! Decomposes the variance of the objective function into contributions -//! from individual parameters (main effects) and parameter interactions. +//! fANOVA decomposes the variance of the objective function into +//! contributions from individual parameters (**main effects**) and +//! parameter pairs (**interaction effects**). This helps answer the +//! question: *"Which parameters matter most, and do any parameters +//! interact?"* //! -//! The algorithm: -//! 1. Fits a random forest to `(parameters) -> objective_value` -//! 2. Applies functional ANOVA decomposition to the forest -//! 3. Computes main effects (single-parameter importance) -//! 4. Computes interaction effects (pairwise parameter importance) +//! # Algorithm +//! +//! 1. Fit a random forest to the mapping `(parameters) → objective` +//! 2. Apply functional ANOVA decomposition to the trained forest +//! 3. Compute main effects: the variance explained by each parameter alone +//! 4. Compute interaction effects: the additional variance explained by +//! pairs of parameters beyond their individual contributions +//! 5. Normalize so all importances sum to 1.0 +//! +//! # When to use +//! +//! - **After optimization**: call [`Study::fanova()`](crate::Study::fanova) +//! or [`Study::fanova_with_config()`](crate::Study::fanova_with_config) +//! to identify which parameters had the most impact +//! - **Interaction detection**: unlike Spearman correlation +//! ([`Study::param_importance()`](crate::Study::param_importance)), +//! fANOVA can detect non-linear relationships and parameter interactions +//! - **Hyperparameter tuning**: focus tuning effort on high-importance +//! parameters and fix low-importance ones to reasonable defaults +//! +//! # Reference +//! +//! Hutter, F., Hoos, H. & Leyton-Brown, K. (2014). "An Efficient +//! Approach for Assessing Hyperparameter Importance." ICML 2014. +//! +//! # Example +//! +//! ``` +//! use optimizer::prelude::*; +//! +//! let study: Study<f64> = Study::new(Direction::Minimize); +//! let x = FloatParam::new(0.0, 10.0).name("x"); +//! let y = FloatParam::new(0.0, 10.0).name("y"); +//! +//! study +//! .optimize(50, |trial| { +//! let xv = x.suggest(trial)?; +//! let yv = y.suggest(trial)?; +//! // x matters much more than y +//! Ok::<_, optimizer::Error>(3.0 * xv + 0.1 * yv) +//! }) +//! .unwrap(); +//! +//! let result = study.fanova().unwrap(); +//! // Main effects sorted by descending importance +//! assert_eq!(result.main_effects[0].0, "x"); +//! ``` /// Result of fANOVA analysis. +/// +/// All importance values are fractions of total variance and sum to 1.0 +/// across main effects and interactions combined. #[derive(Debug, Clone)] pub struct FanovaResult { /// Per-parameter importance (fraction of total variance explained). - /// Sorted by descending importance. + /// + /// Sorted by descending importance. Each entry is + /// `(parameter_name, importance)` where importance is in `[0.0, 1.0]`. pub main_effects: Vec<(String, f64)>, /// Pairwise interaction importance (fraction of total variance explained). - /// Sorted by descending importance. + /// + /// Sorted by descending importance. Each entry is + /// `((param_a, param_b), importance)`. Only pairs with non-negligible + /// interaction (> 1e-10) are included. pub interactions: Vec<((String, String), f64)>, } /// Configuration for fANOVA analysis. +/// +/// Use [`Default::default()`] for reasonable settings, or customize +/// the random forest parameters for specific needs. Pass to +/// [`Study::fanova_with_config()`](crate::Study::fanova_with_config). #[derive(Debug, Clone)] pub struct FanovaConfig { /// Number of trees in the random forest (default: 64). diff --git a/src/importance.rs b/src/importance.rs index 8590302..90b6e76 100644 --- a/src/importance.rs +++ b/src/importance.rs @@ -1,4 +1,26 @@ //! Parameter importance via Spearman rank correlation. +//! +//! Compute the absolute Spearman rank correlation between each parameter +//! and the objective value to estimate which parameters most influence +//! the outcome. This is a lightweight, non-parametric alternative to +//! [`fANOVA`](crate::fanova) that works well for monotonic relationships. +//! +//! # How it works +//! +//! 1. Rank parameter values and objective values independently +//! 2. Compute the Pearson correlation on the ranks (= Spearman ρ) +//! 3. Take the absolute value (direction of correlation is not relevant +//! for importance) +//! +//! # When to use +//! +//! - **Quick importance check**: call +//! [`Study::param_importance()`](crate::Study::param_importance) after +//! optimization for a fast, interpretable ranking +//! - **Monotonic relationships**: Spearman captures monotonic (not just +//! linear) correlations but may miss non-monotonic effects or interactions +//! - For interaction detection or non-linear importance, use +//! [`fANOVA`](crate::fanova) instead /// Assign average ranks to a slice of `f64` values (handles ties). #[allow(clippy::cast_precision_loss, clippy::float_cmp)] diff --git a/src/lib.rs b/src/lib.rs index 1d79dcd..8df4aec 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,194 +9,79 @@ #![deny(clippy::pedantic)] #![deny(clippy::std_instead_of_core)] -//! A black-box optimization library with multiple sampling strategies. +//! Bayesian and population-based optimization library with an Optuna-like API +//! for hyperparameter tuning and black-box optimization. It ships 12 samplers +//! (from random search to CMA-ES and NSGA-III), 8 pruners, async/parallel +//! evaluation, and optional journal-based persistence — all with zero required +//! feature flags for the common case. //! -//! This library provides an Optuna-like API for hyperparameter optimization -//! with support for multiple sampling algorithms: +//! # Getting Started //! -//! - **Random Search** - Simple random sampling for baseline comparisons -//! - **TPE (Tree-Parzen Estimator)** - Bayesian optimization for efficient search -//! - **Grid Search** - Exhaustive search over a specified parameter grid -//! - **Sobol (QMC)** - Quasi-random sampling for better space coverage (requires `sobol` feature) -//! - **CMA-ES** - Covariance Matrix Adaptation Evolution Strategy for continuous optimization (requires `cma-es` feature) -//! - **DE** - Differential Evolution for population-based global optimization -//! - **GP** - Gaussian Process Bayesian optimization with Expected Improvement (requires `gp` feature) -//! - **BOHB** - Bayesian Optimization + `HyperBand` for budget-aware TPE sampling -//! - **NSGA-II** - Non-dominated Sorting Genetic Algorithm II for multi-objective optimization -//! - **NSGA-III** - Reference-point-based NSGA for many-objective (3+) optimization -//! - **MOEA/D** - Decomposition-based multi-objective with Tchebycheff, Weighted Sum, or PBI -//! - **MOTPE** - Multi-Objective Tree-Parzen Estimator for Bayesian multi-objective optimization -//! -//! Additional features include: -//! -//! - Float, integer, and categorical parameter types -//! - Log-scale and stepped parameter sampling -//! - Synchronous and async optimization -//! - Parallel trial evaluation with bounded concurrency -//! -//! # Quick Start +//! Minimize a function in five lines — no feature flags needed: //! //! ``` //! use optimizer::prelude::*; //! -//! // Create a study with TPE sampler -//! let sampler = TpeSampler::builder().seed(42).build().unwrap(); -//! let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler); -//! -//! // Define parameter search space +//! let study: Study<f64> = Study::new(Direction::Minimize); //! let x = FloatParam::new(-10.0, 10.0).name("x"); //! -//! // Optimize x^2 for 20 trials //! study -//! .optimize(20, |trial| { -//! let x_val = x.suggest(trial)?; -//! Ok::<_, Error>(x_val * x_val) +//! .optimize(50, |trial| { +//! let v = x.suggest(trial)?; +//! Ok::<_, Error>((v - 3.0).powi(2)) //! }) //! .unwrap(); //! -//! // Get the best result //! let best = study.best_trial().unwrap(); -//! println!("x = {}", best.get(&x).unwrap()); +//! println!("x = {:.4}, f(x) = {:.4}", best.get(&x).unwrap(), best.value); //! ``` //! -//! # Creating a Study +//! # Core Concepts //! -//! A [`Study`] manages optimization trials. Create one with an optimization direction: +//! | Type | Role | +//! |------|------| +//! | [`Study`] | Drive an optimization loop: create trials, record results, track the best. | +//! | [`Trial`] | A single evaluation of the objective function, carrying suggested parameter values. | +//! | [`Parameter`] | Define the search space — [`FloatParam`], [`IntParam`], [`CategoricalParam`], [`BoolParam`], [`EnumParam`]. | +//! | [`Sampler`](sampler::Sampler) | Strategy for choosing the next point to evaluate (TPE, CMA-ES, random, etc.). | +//! | [`Direction`] | Whether the study minimizes or maximizes the objective value. | //! -//! ``` -//! use optimizer::sampler::random::RandomSampler; -//! use optimizer::sampler::tpe::TpeSampler; -//! use optimizer::{Direction, Study}; +//! # Sampler Guide //! -//! // Minimize with default random sampler -//! let study: Study<f64> = Study::new(Direction::Minimize); +//! ## Single-objective samplers //! -//! // Maximize with TPE sampler -//! let study: Study<f64> = Study::with_sampler(Direction::Maximize, TpeSampler::new()); +//! | Sampler | Algorithm | Best for | Feature flag | +//! |---------|-----------|----------|--------------| +//! | [`RandomSampler`] | Uniform random | Baselines, high-dimensional | — | +//! | [`TpeSampler`] | Tree-Parzen Estimator | General-purpose Bayesian | — | +//! | [`GridSearchSampler`] | Exhaustive grid | Small, discrete spaces | — | +//! | [`SobolSampler`] | Sobol quasi-random sequence | Space-filling, low dimensions | `sobol` | +//! | [`CmaEsSampler`] | CMA-ES | Continuous, moderate dimensions | `cma-es` | +//! | [`GpSampler`] | Gaussian Process + EI | Expensive objectives, few trials | `gp` | +//! | [`DifferentialEvolutionSampler`] | Differential Evolution | Non-convex, population-based | — | +//! | [`BohbSampler`] | BOHB (TPE + `HyperBand`) | Budget-aware early stopping | — | //! -//! // With seeded sampler for reproducibility -//! let study: Study<f64> = Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(42)); -//! ``` +//! ## Multi-objective samplers //! -//! # Suggesting Parameters -//! -//! Within the objective function, use parameter types to suggest values: -//! -//! ``` -//! use optimizer::parameter::{BoolParam, CategoricalParam, FloatParam, IntParam, Parameter}; -//! use optimizer::{Direction, Study}; -//! -//! let study: Study<f64> = Study::new(Direction::Minimize); -//! -//! // Define parameter search spaces -//! let x_param = FloatParam::new(0.0, 1.0); -//! let lr_param = FloatParam::new(1e-5, 1e-1).log_scale(); -//! let step_param = FloatParam::new(0.0, 1.0).step(0.1); -//! let n_param = IntParam::new(1, 10); -//! let batch_param = IntParam::new(16, 256).log_scale(); -//! let units_param = IntParam::new(32, 512).step(32); -//! let flag_param = BoolParam::new(); -//! let optimizer_param = CategoricalParam::new(vec!["sgd", "adam", "rmsprop"]); -//! -//! study -//! .optimize(10, |trial| { -//! let x = x_param.suggest(trial)?; -//! let lr = lr_param.suggest(trial)?; -//! let step = step_param.suggest(trial)?; -//! let n = n_param.suggest(trial)?; -//! let batch = batch_param.suggest(trial)?; -//! let units = units_param.suggest(trial)?; -//! let flag = flag_param.suggest(trial)?; -//! let optimizer = optimizer_param.suggest(trial)?; -//! -//! Ok::<_, optimizer::Error>(x * n as f64) -//! }) -//! .unwrap(); -//! ``` -//! -//! # Available Samplers -//! -//! ## Random Search -//! -//! The simplest sampling strategy, useful for baselines: -//! -//! ``` -//! use optimizer::sampler::random::RandomSampler; -//! use optimizer::{Direction, Study}; -//! -//! let study: Study<f64> = Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(42)); -//! ``` -//! -//! ## TPE (Tree-Parzen Estimator) -//! -//! Bayesian optimization that learns from previous trials: -//! -//! ``` -//! use optimizer::sampler::tpe::TpeSampler; -//! -//! let sampler = TpeSampler::builder() -//! .gamma(0.15) // Quantile for good/bad split -//! .n_startup_trials(20) // Random trials before TPE -//! .n_ei_candidates(32) // Candidates to evaluate -//! .seed(42) // Reproducibility -//! .build() -//! .unwrap(); -//! ``` -//! -//! ## Grid Search -//! -//! Exhaustive search over a discretized parameter space: -//! -//! ``` -//! use optimizer::sampler::grid::GridSearchSampler; -//! use optimizer::{Direction, Study}; -//! -//! let sampler = GridSearchSampler::builder() -//! .n_points_per_param(10) // Points per parameter dimension -//! .build(); -//! -//! let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler); -//! ``` -//! -//! # Async and Parallel Optimization -//! -//! With the `async` feature enabled, you can run trials asynchronously: -//! -//! ```ignore -//! use optimizer::{Study, Direction}; -//! use optimizer::parameter::{FloatParam, Parameter}; -//! -//! let x_param = FloatParam::new(0.0, 1.0); -//! -//! // Sequential async -//! study.optimize_async(10, |mut trial| { -//! let x_param = x_param.clone(); -//! async move { -//! let x = x_param.suggest(&mut trial)?; -//! Ok((trial, x * x)) -//! } -//! }).await?; -//! -//! // Parallel with bounded concurrency -//! study.optimize_parallel(10, 4, |mut trial| { -//! let x_param = x_param.clone(); -//! async move { -//! let x = x_param.suggest(&mut trial)?; -//! Ok((trial, x * x)) -//! } -//! }).await?; -//! ``` +//! | Sampler | Algorithm | Best for | Feature flag | +//! |---------|-----------|----------|--------------| +//! | [`Nsga2Sampler`] | NSGA-II | 2-3 objectives | — | +//! | [`Nsga3Sampler`] | NSGA-III (reference-point) | 3+ objectives | — | +//! | [`MoeadSampler`] | MOEA/D (decomposition) | Many objectives, structured fronts | — | +//! | [`MotpeSampler`] | Multi-Objective TPE | Bayesian multi-objective | — | //! //! # Feature Flags //! -//! - `async`: Enable async optimization methods (requires tokio) -//! - `derive`: Enable `#[derive(Categorical)]` for enum parameters -//! - `serde`: Enable `Serialize`/`Deserialize` on public types and `Study::save()`/`Study::load()` -//! - `sobol`: Enable the Sobol quasi-random sampler for better space coverage -//! - `cma-es`: Enable the CMA-ES sampler for continuous optimization -//! - `gp`: Enable the Gaussian Process sampler for Bayesian optimization -//! - `visualization`: Generate self-contained HTML reports with interactive Plotly.js charts -//! - `tracing`: Emit structured log events via the [`tracing`](https://docs.rs/tracing) crate at key optimization points +//! | Flag | What it enables | Default | +//! |------|----------------|---------| +//! | `async` | Async/parallel optimization via tokio ([`Study::optimize_async`], [`Study::optimize_parallel`]) | off | +//! | `derive` | `#[derive(Categorical)]` for enum parameters | off | +//! | `serde` | `Serialize`/`Deserialize` on public types, [`Study::save`]/[`Study::load`] | off | +//! | `journal` | [`JournalStorage`] — JSONL persistence with file locking (enables `serde`) | off | +//! | `sobol` | [`SobolSampler`] — quasi-random low-discrepancy sequences | off | +//! | `cma-es` | [`CmaEsSampler`] — Covariance Matrix Adaptation Evolution Strategy | off | +//! | `gp` | [`GpSampler`] — Gaussian Process surrogate with Expected Improvement | off | +//! | `tracing` | Structured log events via [`tracing`](https://docs.rs/tracing) at key optimization points | off | /// Emit a `tracing::info!` event when the `tracing` feature is enabled. /// No-op otherwise. diff --git a/src/multi_objective.rs b/src/multi_objective.rs index 5a4b3b8..d47ce37 100644 --- a/src/multi_objective.rs +++ b/src/multi_objective.rs @@ -1,9 +1,28 @@ //! Multi-objective optimization via a dedicated study type. //! -//! [`MultiObjectiveStudy`] manages trials that return multiple objective -//! values. It supports arbitrary numbers of objectives with per-objective -//! directions (minimize or maximize). Use [`pareto_front()`](MultiObjectiveStudy::pareto_front) -//! to retrieve the Pareto-optimal solutions. +//! [`MultiObjectiveStudy`] manages trials that return **multiple** objective +//! values simultaneously. It supports arbitrary numbers of objectives with +//! per-objective directions (minimize or maximize). +//! +//! # Key concepts +//! +//! In multi-objective optimization there is usually no single best solution. +//! Instead, there is a **Pareto front** — the set of solutions where no +//! objective can be improved without worsening another. Use +//! [`pareto_front()`](MultiObjectiveStudy::pareto_front) to retrieve these +//! non-dominated solutions after optimization. +//! +//! A solution **dominates** another if it is at least as good in all +//! objectives and strictly better in at least one. Solutions that are not +//! dominated by any other are called **Pareto-optimal**. +//! +//! # Samplers +//! +//! By default a random sampler is used. For smarter search, pass a +//! [`MultiObjectiveSampler`] such as [`Nsga2Sampler`](crate::Nsga2Sampler), +//! [`Nsga3Sampler`](crate::Nsga3Sampler), or +//! [`MoeadSampler`](crate::MoeadSampler) via +//! [`MultiObjectiveStudy::with_sampler`]. //! //! # Examples //! @@ -46,6 +65,11 @@ use crate::types::{Direction, TrialState}; // --------------------------------------------------------------------------- /// A completed trial with multiple objective values. +/// +/// Each trial stores its sampled parameter values, the vector of +/// objective values (one per objective), and optional constraint values. +/// Retrieve typed parameter values with [`get()`](Self::get) and check +/// constraint feasibility with [`is_feasible()`](Self::is_feasible). #[derive(Clone, Debug)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct MultiObjectiveTrial { @@ -111,9 +135,14 @@ impl MultiObjectiveTrial { /// Trait for samplers aware of multi-objective history. /// -/// Separate from [`Sampler`] because NSGA-II needs access to -/// `&[MultiObjectiveTrial]` (with vector-valued objectives) and -/// `&[Direction]` (one direction per objective). +/// Separate from [`Sampler`] because multi-objective algorithms (e.g., +/// NSGA-II) need access to the full vector of objective values per trial +/// (`&[MultiObjectiveTrial]`) and the per-objective directions +/// (`&[Direction]`). +/// +/// Implementations include [`Nsga2Sampler`](crate::Nsga2Sampler), +/// [`Nsga3Sampler`](crate::Nsga3Sampler), and +/// [`MoeadSampler`](crate::MoeadSampler). pub trait MultiObjectiveSampler: Send + Sync { /// Samples a parameter value from the given distribution. fn sample( @@ -181,9 +210,12 @@ impl Sampler for MoSamplerBridge { /// A study for multi-objective optimization. /// -/// Manages trials that return multiple objective values. Supports +/// Manage trials that return multiple objective values. Supports /// arbitrary numbers of objectives with independent minimize/maximize -/// directions. +/// directions. After optimization, call [`pareto_front()`](Self::pareto_front) +/// to retrieve the non-dominated solutions. +/// +/// For single-objective optimization, use [`Study`](crate::Study) instead. /// /// # Examples /// @@ -269,7 +301,11 @@ impl MultiObjectiveStudy { self.completed_trials.read().clone() } - /// Returns the Pareto-optimal trials (front 0). + /// Return the Pareto-optimal trials (the non-dominated front). + /// + /// Uses fast non-dominated sorting (Deb et al., 2002) from the + /// [`pareto`](crate::pareto) module. Returns an empty vec if no + /// trials have completed. #[must_use] pub fn pareto_front(&self) -> Vec<MultiObjectiveTrial> { let trials = self.completed_trials.read(); diff --git a/src/param.rs b/src/param.rs index 6dd3e60..864a19e 100644 --- a/src/param.rs +++ b/src/param.rs @@ -1,18 +1,35 @@ -//! Parameter value storage types. +//! Raw parameter value storage. +//! +//! [`ParamValue`] is the type-erased representation of a sampled parameter. +//! Users rarely construct `ParamValue` directly — the +//! [`Parameter::suggest`](crate::parameter::Parameter::suggest) method returns +//! the already-typed value (e.g., `f64` for [`FloatParam`](crate::parameter::FloatParam)). +//! +//! `ParamValue` is useful when inspecting raw trial data via +//! [`Trial::params`](crate::Trial::params) or +//! [`CompletedTrial::params`](crate::sampler::CompletedTrial). -/// Represents a sampled parameter value. +/// A type-erased sampled parameter value. /// -/// This enum stores different parameter value types uniformly. -/// For categorical parameters, the `Categorical` variant stores -/// the index into the choices array. +/// Stores float, integer, or categorical (index) values uniformly. +/// For categorical parameters the `Categorical` variant stores the +/// zero-based index into the choices array, not the choice itself. +/// +/// # Display +/// +/// `ParamValue` implements [`Display`](core::fmt::Display): floats and +/// integers print their numeric value, and categoricals print `category(i)`. #[derive(Clone, Debug, PartialEq)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum ParamValue { - /// A floating-point parameter value. + /// A floating-point parameter value (from [`FloatParam`](crate::parameter::FloatParam)). Float(f64), - /// An integer parameter value. + /// An integer parameter value (from [`IntParam`](crate::parameter::IntParam)). Int(i64), - /// A categorical parameter value, stored as an index into the choices array. + /// A categorical index into the choices array (from + /// [`CategoricalParam`](crate::parameter::CategoricalParam), + /// [`BoolParam`](crate::parameter::BoolParam), or + /// [`EnumParam`](crate::parameter::EnumParam)). Categorical(usize), } diff --git a/src/parameter.rs b/src/parameter.rs index 34f5fd3..6c48931 100644 --- a/src/parameter.rs +++ b/src/parameter.rs @@ -1,8 +1,19 @@ -//! Central parameter trait and built-in parameter types. +//! Parameter trait and five built-in parameter types. //! -//! The [`Parameter`] trait provides a unified way to define parameter types -//! and suggest values from a [`Trial`]. Built-in implementations -//! cover floats, integers, categoricals, booleans, and enum types. +//! The [`Parameter`] trait provides a unified way to define search-space +//! dimensions and sample values from a [`Trial`]. Five implementations +//! cover the most common hyperparameter types: +//! +//! | Type | Sampled value | Typical use | +//! |------|---------------|-------------| +//! | [`FloatParam`] | `f64` | Learning rate, dropout probability | +//! | [`IntParam`] | `i64` | Layer count, batch size | +//! | [`CategoricalParam`] | `T: Clone` | Optimizer name, activation function | +//! | [`BoolParam`] | `bool` | Feature toggle | +//! | [`EnumParam`] | `T: Categorical` | Typed enum variant selection | +//! +//! All five types support `.name()` for a human-readable label and +//! `.suggest(&mut trial)` as a shorthand for `trial.suggest_param(¶m)`. //! //! # Example //! @@ -14,10 +25,17 @@ //! //! let lr = FloatParam::new(1e-5, 1e-1) //! .log_scale() +//! .name("learning_rate") +//! .suggest(&mut trial) +//! .unwrap(); +//! let layers = IntParam::new(1, 10) +//! .name("n_layers") +//! .suggest(&mut trial) +//! .unwrap(); +//! let dropout = BoolParam::new() +//! .name("use_dropout") //! .suggest(&mut trial) //! .unwrap(); -//! let layers = IntParam::new(1, 10).suggest(&mut trial).unwrap(); -//! let dropout = BoolParam::new().suggest(&mut trial).unwrap(); //! ``` use core::fmt::Debug; @@ -42,7 +60,7 @@ static NEXT_PARAM_ID: AtomicU64 = AtomicU64::new(0); pub struct ParamId(u64); impl ParamId { - /// Creates a new unique `ParamId`. + /// Create a new unique `ParamId`. pub fn new() -> Self { Self(NEXT_PARAM_ID.fetch_add(1, Ordering::Relaxed)) } @@ -60,52 +78,67 @@ impl core::fmt::Display for ParamId { } } -/// A trait for defining parameter types that can be suggested by a [`Trial`]. +/// Define a parameter type that can be suggested by a [`Trial`]. /// /// Implementors specify the distribution to sample from and how to convert -/// the raw [`ParamValue`] back into a typed value. +/// the raw [`ParamValue`] back into a typed value. See the five built-in +/// implementations: [`FloatParam`], [`IntParam`], [`CategoricalParam`], +/// [`BoolParam`], and [`EnumParam`]. pub trait Parameter: Debug { /// The typed value returned after sampling. type Value; - /// Returns the unique identifier for this parameter. + /// Return the unique identifier for this parameter. fn id(&self) -> ParamId; - /// Returns the distribution that this parameter samples from. + /// Return the distribution that this parameter samples from. fn distribution(&self) -> Distribution; - /// Converts a raw [`ParamValue`] into the typed value. + /// Convert a raw [`ParamValue`] into the typed value. /// /// # Errors /// - /// Returns an error if the `ParamValue` variant doesn't match what this parameter expects. + /// Return an error if the `ParamValue` variant does not match what this parameter expects. fn cast_param_value(&self, param_value: &ParamValue) -> Result<Self::Value>; - /// Validates the parameter configuration. + /// Validate the parameter configuration. /// /// Called before sampling. The default implementation accepts all configurations. /// /// # Errors /// - /// Returns an error if the parameter configuration is invalid. + /// Return an error if the parameter configuration is invalid. fn validate(&self) -> Result<()> { Ok(()) } - /// Returns a human-readable label for this parameter. + /// Return a human-readable label for this parameter. /// - /// Defaults to the `Debug` output of the parameter. + /// Defaults to the `Debug` output of the parameter. Override with + /// the `.name()` builder method on concrete types. fn label(&self) -> String { format!("{self:?}") } - /// Suggests a value for this parameter from the given trial. + /// Suggest a value for this parameter from the given trial. /// /// This is a convenience method that delegates to [`Trial::suggest_param`]. /// + /// # Examples + /// + /// ``` + /// use optimizer::Trial; + /// use optimizer::parameter::{FloatParam, Parameter}; + /// + /// let mut trial = Trial::new(0); + /// let param = FloatParam::new(-5.0, 5.0).name("x"); + /// let value: f64 = param.suggest(&mut trial).unwrap(); + /// assert!((-5.0..=5.0).contains(&value)); + /// ``` + /// /// # Errors /// - /// Returns an error if validation fails, the parameter conflicts with + /// Return an error if validation fails, the parameter conflicts with /// a previously suggested parameter of the same id, or sampling fails. fn suggest(&self, trial: &mut Trial) -> Result<Self::Value> where @@ -117,7 +150,7 @@ pub trait Parameter: Debug { /// A floating-point parameter with optional log-scale and step size. /// -/// # Example +/// # Examples /// /// ``` /// use optimizer::Trial; @@ -128,13 +161,14 @@ pub trait Parameter: Debug { /// // Simple range /// let x = FloatParam::new(0.0, 1.0).suggest(&mut trial).unwrap(); /// -/// // Log-scale +/// // Log-scale with a human-readable name /// let lr = FloatParam::new(1e-5, 1e-1) /// .log_scale() +/// .name("learning_rate") /// .suggest(&mut trial) /// .unwrap(); /// -/// // Stepped +/// // Stepped (values will be multiples of 0.25) /// let step = FloatParam::new(0.0, 1.0) /// .step(0.25) /// .suggest(&mut trial) @@ -151,7 +185,7 @@ pub struct FloatParam { } impl FloatParam { - /// Creates a new float parameter with the given bounds. + /// Create a new float parameter sampling uniformly from `[low, high]`. #[must_use] pub fn new(low: f64, high: f64) -> Self { Self { @@ -164,21 +198,21 @@ impl FloatParam { } } - /// Enables log-scale sampling. + /// Enable log-scale sampling (bounds must be positive). #[must_use] pub fn log_scale(mut self) -> Self { self.log_scale = true; self } - /// Sets a step size for discretized sampling. + /// Set a step size for discretized sampling. #[must_use] pub fn step(mut self, step: f64) -> Self { self.step = Some(step); self } - /// Sets a human-readable name for this parameter. + /// Set a human-readable name for this parameter. /// /// When set, this name is used as the parameter's label instead of /// the default `Debug` output. @@ -245,7 +279,7 @@ impl Parameter for FloatParam { /// An integer parameter with optional log-scale and step size. /// -/// # Example +/// # Examples /// /// ``` /// use optimizer::Trial; @@ -254,15 +288,19 @@ impl Parameter for FloatParam { /// let mut trial = Trial::new(0); /// /// // Simple range -/// let n = IntParam::new(1, 10).suggest(&mut trial).unwrap(); +/// let n = IntParam::new(1, 10) +/// .name("n_layers") +/// .suggest(&mut trial) +/// .unwrap(); /// /// // Log-scale /// let batch = IntParam::new(1, 1024) /// .log_scale() +/// .name("batch_size") /// .suggest(&mut trial) /// .unwrap(); /// -/// // Stepped +/// // Stepped (multiples of 32) /// let units = IntParam::new(32, 512).step(32).suggest(&mut trial).unwrap(); /// ``` #[derive(Clone, Debug)] @@ -276,7 +314,7 @@ pub struct IntParam { } impl IntParam { - /// Creates a new integer parameter with the given bounds. + /// Create a new integer parameter sampling uniformly from `[low, high]`. #[must_use] pub fn new(low: i64, high: i64) -> Self { Self { @@ -289,21 +327,21 @@ impl IntParam { } } - /// Enables log-scale sampling. + /// Enable log-scale sampling (bounds must be ≥ 1). #[must_use] pub fn log_scale(mut self) -> Self { self.log_scale = true; self } - /// Sets a step size for discretized sampling. + /// Set a step size for discretized sampling. #[must_use] pub fn step(mut self, step: i64) -> Self { self.step = Some(step); self } - /// Sets a human-readable name for this parameter. + /// Set a human-readable name for this parameter. /// /// When set, this name is used as the parameter's label instead of /// the default `Debug` output. @@ -370,7 +408,10 @@ impl Parameter for IntParam { /// A categorical parameter that selects from a list of choices. /// -/// # Example +/// The generic type `T` is the element type of the choices vector. +/// The sampler picks an index and the corresponding element is returned. +/// +/// # Examples /// /// ``` /// use optimizer::Trial; @@ -378,6 +419,7 @@ impl Parameter for IntParam { /// /// let mut trial = Trial::new(0); /// let opt = CategoricalParam::new(vec!["sgd", "adam", "rmsprop"]) +/// .name("optimizer") /// .suggest(&mut trial) /// .unwrap(); /// ``` @@ -389,7 +431,7 @@ pub struct CategoricalParam<T: Clone> { } impl<T: Clone> CategoricalParam<T> { - /// Creates a new categorical parameter with the given choices. + /// Create a new categorical parameter with the given choices. #[must_use] pub fn new(choices: Vec<T>) -> Self { Self { @@ -399,7 +441,7 @@ impl<T: Clone> CategoricalParam<T> { } } - /// Sets a human-readable name for this parameter. + /// Set a human-readable name for this parameter. /// /// When set, this name is used as the parameter's label instead of /// the default `Debug` output. @@ -444,16 +486,19 @@ impl<T: Clone + Debug> Parameter for CategoricalParam<T> { } } -/// A boolean parameter (equivalent to a categorical with `[false, true]`). +/// A boolean parameter (equivalent to a two-choice categorical: `false` / `true`). /// -/// # Example +/// # Examples /// /// ``` /// use optimizer::Trial; /// use optimizer::parameter::{BoolParam, Parameter}; /// /// let mut trial = Trial::new(0); -/// let dropout = BoolParam::new().suggest(&mut trial).unwrap(); +/// let use_dropout = BoolParam::new() +/// .name("use_dropout") +/// .suggest(&mut trial) +/// .unwrap(); /// ``` #[derive(Clone, Debug)] pub struct BoolParam { @@ -462,7 +507,7 @@ pub struct BoolParam { } impl BoolParam { - /// Creates a new boolean parameter. + /// Create a new boolean parameter. #[must_use] pub fn new() -> Self { Self { @@ -471,7 +516,7 @@ impl BoolParam { } } - /// Sets a human-readable name for this parameter. + /// Set a human-readable name for this parameter. /// /// When set, this name is used as the parameter's label instead of /// the default `Debug` output. @@ -513,10 +558,10 @@ impl Parameter for BoolParam { } } -/// A trait for enum types that can be used as categorical parameters. +/// Map an enum type to sequential indices for use as a categorical parameter. /// -/// This trait maps enum variants to sequential indices and back. It can be -/// derived automatically for fieldless enums using `#[derive(Categorical)]` +/// This trait converts enum variants to sequential indices and back. It can +/// be derived automatically for fieldless enums using `#[derive(Categorical)]` /// when the `derive` feature is enabled. /// /// # Example @@ -558,20 +603,24 @@ pub trait Categorical: Sized + Clone { /// The number of variants in the enum. const N_CHOICES: usize; - /// Creates an instance from a variant index. + /// Create an instance from a variant index. /// /// # Panics /// /// Panics if `index >= N_CHOICES`. fn from_index(index: usize) -> Self; - /// Returns the index of this variant. + /// Return the index of this variant. fn to_index(&self) -> usize; } /// A parameter that selects from the variants of an enum implementing [`Categorical`]. /// -/// # Example +/// Prefer this over [`CategoricalParam`] when the choices map to a Rust enum, +/// because the returned value is already the correct variant — no string +/// matching required. +/// +/// # Examples /// /// ``` /// use optimizer::Trial; @@ -604,7 +653,10 @@ pub trait Categorical: Sized + Clone { /// } /// /// let mut trial = Trial::new(0); -/// let opt = EnumParam::<Optimizer>::new().suggest(&mut trial).unwrap(); +/// let opt = EnumParam::<Optimizer>::new() +/// .name("optimizer") +/// .suggest(&mut trial) +/// .unwrap(); /// ``` #[derive(Clone, Debug)] pub struct EnumParam<T: Categorical> { @@ -614,7 +666,7 @@ pub struct EnumParam<T: Categorical> { } impl<T: Categorical> EnumParam<T> { - /// Creates a new enum parameter. + /// Create a new enum parameter over all variants of `T`. #[must_use] pub fn new() -> Self { Self { @@ -624,7 +676,7 @@ impl<T: Categorical> EnumParam<T> { } } - /// Sets a human-readable name for this parameter. + /// Set a human-readable name for this parameter. /// /// When set, this name is used as the parameter's label instead of /// the default `Debug` output. diff --git a/src/pareto.rs b/src/pareto.rs index 4cfc822..c6a23c9 100644 --- a/src/pareto.rs +++ b/src/pareto.rs @@ -1,15 +1,71 @@ //! Pareto front analysis utilities for multi-objective optimization. //! -//! Provides functions for analyzing and working with Pareto fronts: +//! In multi-objective optimization there is generally no single best +//! solution. Instead, the goal is to find the **Pareto front** — the set +//! of solutions where no objective can be improved without worsening +//! another. This module provides tools for computing and analyzing Pareto +//! fronts. //! -//! - [`hypervolume`] — measure the quality of a Pareto front -//! - [`non_dominated_sort`] — rank solutions into successive fronts -//! - [`pareto_front_indices`] — filter to non-dominated solutions only -//! - [`crowding_distance`] — measure diversity within a front +//! # Available functions //! -//! Internally also provides fast non-dominated sorting (Deb et al., 2002) -//! used by [`MultiObjectiveStudy::pareto_front()`](crate::MultiObjectiveStudy::pareto_front) +//! | Function | Purpose | +//! |---|---| +//! | [`hypervolume`] | Measure the quality of a Pareto front (volume of dominated space) | +//! | [`non_dominated_sort`] | Rank solutions into successive fronts (front 0, 1, …) | +//! | [`pareto_front_indices`] | Filter to non-dominated (Pareto-optimal) solutions only | +//! | [`crowding_distance`] | Measure diversity/spread within a single front | +//! +//! # When to use +//! +//! - **Evaluating front quality**: Use [`hypervolume`] to compare two +//! Pareto fronts — a higher hypervolume indicates a better-quality front. +//! - **Ranking all solutions**: Use [`non_dominated_sort`] to partition +//! solutions into successive fronts, useful for selection in evolutionary +//! algorithms. +//! - **Extracting the best solutions**: Use [`pareto_front_indices`] to get +//! only the non-dominated set. +//! - **Diversity measurement**: Use [`crowding_distance`] to quantify how +//! spread out solutions are within a front, which helps maintain diversity. +//! +//! Internally, this module also provides the fast non-dominated sorting +//! algorithm (Deb et al., 2002) used by +//! [`MultiObjectiveStudy::pareto_front()`](crate::MultiObjectiveStudy::pareto_front) //! and [`Nsga2Sampler`](crate::Nsga2Sampler). +//! +//! # Example +//! +//! ``` +//! use optimizer::Direction; +//! use optimizer::pareto::{ +//! crowding_distance, hypervolume, non_dominated_sort, pareto_front_indices, +//! }; +//! +//! let solutions = vec![ +//! vec![1.0, 5.0], // Pareto-optimal +//! vec![5.0, 1.0], // Pareto-optimal +//! vec![3.0, 3.0], // Pareto-optimal +//! vec![4.0, 4.0], // Dominated by (3, 3) +//! ]; +//! let dirs = [Direction::Minimize, Direction::Minimize]; +//! +//! // Non-dominated sorting: front 0 has indices {0, 1, 2} +//! let fronts = non_dominated_sort(&solutions, &dirs); +//! assert_eq!(fronts.len(), 2); +//! +//! // Pareto front indices (shortcut for fronts[0]) +//! let mut front = pareto_front_indices(&solutions, &dirs); +//! front.sort(); +//! assert_eq!(front, vec![0, 1, 2]); +//! +//! // Hypervolume with reference point (6, 6) +//! let front_values: Vec<_> = front.iter().map(|&i| solutions[i].clone()).collect(); +//! let hv = hypervolume(&front_values, &[6.0, 6.0], &dirs); +//! assert!(hv > 0.0); +//! +//! // Crowding distance for diversity analysis +//! let cd = crowding_distance(&front_values, &dirs); +//! assert!(cd[0].is_infinite()); // boundary solution +//! ``` use crate::types::Direction; @@ -200,12 +256,17 @@ pub(crate) fn crowding_distance_indexed(front_indices: &[usize], values: &[Vec<f /// Compute the hypervolume indicator of a Pareto front. /// /// The hypervolume is the volume of the objective space dominated by -/// the Pareto front and bounded by a reference point. Higher values -/// indicate a better front. +/// the Pareto front and bounded by a reference point. A **higher** +/// hypervolume indicates a better front (closer to the ideal and more +/// spread out). /// /// Each entry in `front` is one solution's objective values. /// `reference_point` should be worse than all front members in every -/// objective (e.g., the worst acceptable values). +/// objective (e.g., the worst acceptable values). Solutions that do +/// not strictly dominate the reference point are ignored. +/// +/// Uses recursive slicing for dimensions > 1. Complexity grows with +/// the number of objectives and front size. /// /// # Panics /// @@ -345,12 +406,14 @@ fn non_dominated_minimize(points: &[Vec<f64>]) -> Vec<Vec<f64>> { /// Compute non-dominated sorting of a set of solutions. /// -/// Returns a vec of fronts, where `fronts[0]` is the Pareto front, -/// `fronts[1]` is the next best, etc. Each inner vec contains indices -/// into the original `solutions` slice. +/// Return a vec of fronts, where `fronts[0]` is the Pareto front +/// (non-dominated solutions), `fronts[1]` is the next-best front +/// (dominated only by front 0), and so on. Each inner vec contains +/// indices into the original `solutions` slice. /// -/// Uses the fast non-dominated sorting algorithm from -/// Deb et al. (2002) with O(M N²) complexity. +/// Use the fast non-dominated sorting algorithm from +/// Deb et al. (2002) with O(M × N²) complexity, where M is the +/// number of objectives and N is the number of solutions. #[must_use] pub fn non_dominated_sort(solutions: &[Vec<f64>], directions: &[Direction]) -> Vec<Vec<usize>> { fast_non_dominated_sort(solutions, directions) @@ -359,7 +422,8 @@ pub fn non_dominated_sort(solutions: &[Vec<f64>], directions: &[Direction]) -> V /// Filter solutions to return only non-dominated (Pareto-optimal) indices. /// /// Equivalent to `non_dominated_sort(solutions, directions)[0]` but -/// communicates the intent more clearly. +/// communicates the intent more clearly. Use this when you only need +/// the Pareto front and not the full ranking. #[must_use] pub fn pareto_front_indices(solutions: &[Vec<f64>], directions: &[Direction]) -> Vec<usize> { let fronts = fast_non_dominated_sort(solutions, directions); @@ -368,10 +432,13 @@ pub fn pareto_front_indices(solutions: &[Vec<f64>], directions: &[Direction]) -> /// Compute crowding distance for diversity measurement. /// -/// Returns one distance value per solution in `front` (same order). +/// Return one distance value per solution in `front` (same order). /// Boundary solutions (best/worst in any objective) receive /// [`f64::INFINITY`]. Interior solutions get a finite positive value -/// proportional to the gap between their neighbors. +/// proportional to the gap between their neighbors in each objective. +/// +/// Crowding distance is used by NSGA-II to prefer well-spread +/// solutions when two solutions are in the same front. /// /// `directions` is accepted for API consistency but does not affect /// the result, since crowding distance measures spacing regardless of diff --git a/src/pruner/hyperband.rs b/src/pruner/hyperband.rs index 98c6cca..ac82acf 100644 --- a/src/pruner/hyperband.rs +++ b/src/pruner/hyperband.rs @@ -1,3 +1,47 @@ +//! `HyperBand` pruner — adaptive budget scheduling with multiple SHA brackets. +//! +//! `HyperBand` addresses the main weakness of +//! [`SuccessiveHalvingPruner`](super::SuccessiveHalvingPruner): sensitivity to +//! the `min_resource` setting. It runs multiple Successive Halving brackets in +//! parallel, each with a different trade-off between the number of trials and +//! the starting budget: +//! +//! - **Bracket 0**: many trials, small starting budget (aggressive early pruning) +//! - **Bracket `s_max`**: few trials, full budget (no pruning) +//! +//! Trials are assigned to brackets in round-robin order. This ensures that +//! the overall search is robust regardless of how informative early steps are. +//! +//! # When to use +//! +//! - When you don't know how many epochs/steps are needed before performance +//! becomes predictive +//! - As a drop-in upgrade over [`SuccessiveHalvingPruner`](super::SuccessiveHalvingPruner) +//! when you can afford more total trials +//! - For large-scale hyperparameter searches where compute savings matter most +//! +//! # Configuration +//! +//! | Option | Default | Description | +//! |--------|---------|-------------| +//! | `min_resource` | 1 | Smallest budget for the most aggressive bracket | +//! | `max_resource` | 81 | Full budget (last rung in every bracket) | +//! | `reduction_factor` | 3 | At each rung, keep top 1/η trials | +//! | `direction` | `Minimize` | Optimization direction | +//! +//! # Example +//! +//! ``` +//! use optimizer::Direction; +//! use optimizer::pruner::HyperbandPruner; +//! +//! let pruner = HyperbandPruner::new() +//! .min_resource(1) +//! .max_resource(81) +//! .reduction_factor(3) +//! .direction(Direction::Minimize); +//! ``` + use core::sync::atomic::{AtomicU64, Ordering}; use std::collections::HashMap; use std::sync::Mutex; @@ -6,7 +50,7 @@ use super::Pruner; use crate::sampler::CompletedTrial; use crate::types::{Direction, TrialState}; -/// Hyperband pruner that manages multiple Successive Halving brackets. +/// `HyperBand` pruner that manages multiple Successive Halving brackets. /// /// Hyperband addresses SHA's sensitivity to the `min_resource` choice by /// running multiple brackets, each with a different tradeoff between the diff --git a/src/pruner/median.rs b/src/pruner/median.rs index 122d483..0e97c8f 100644 --- a/src/pruner/median.rs +++ b/src/pruner/median.rs @@ -1,3 +1,39 @@ +//! Median pruner — the recommended default pruner for most use cases. +//! +//! At each step, the current trial's intermediate value is compared against +//! the median of all completed trials' values at the same step. Trials +//! performing worse than the median are pruned. +//! +//! This is a convenience wrapper around [`PercentilePruner`](super::PercentilePruner) +//! with a fixed percentile of 50%. +//! +//! # When to use +//! +//! - **Default choice** for any iterative objective (e.g., neural network training) +//! - Works well when intermediate values are a reasonable proxy for final performance +//! - Prunes roughly half of unpromising trials, giving a good speed/accuracy balance +//! +//! If your intermediate values are noisy, consider [`WilcoxonPruner`](super::WilcoxonPruner) +//! or wrapping this pruner in a [`PatientPruner`](super::PatientPruner). +//! +//! # Configuration +//! +//! | Option | Default | Description | +//! |--------|---------|-------------| +//! | `n_warmup_steps` | 0 | Skip pruning in the first N steps | +//! | `n_min_trials` | 1 | Require at least N completed trials before pruning | +//! +//! # Example +//! +//! ``` +//! use optimizer::Direction; +//! use optimizer::pruner::MedianPruner; +//! +//! let pruner = MedianPruner::new(Direction::Minimize) +//! .n_warmup_steps(5) +//! .n_min_trials(3); +//! ``` + use super::Pruner; use super::percentile::compute_percentile; use crate::sampler::CompletedTrial; diff --git a/src/pruner/mod.rs b/src/pruner/mod.rs index 614dfa1..03d7f01 100644 --- a/src/pruner/mod.rs +++ b/src/pruner/mod.rs @@ -3,6 +3,44 @@ //! Pruners decide whether to stop (prune) a trial early based on its //! intermediate values compared to other trials. This is useful for //! discarding unpromising trials before they complete, saving compute. +//! +//! # How pruning works +//! +//! During optimization, each trial reports intermediate values at discrete +//! steps (e.g., validation loss after each training epoch). A pruner inspects +//! these values and compares them against completed trials to decide whether +//! the current trial should be stopped early. +//! +//! The typical flow is: +//! +//! 1. Call [`Trial::report`](crate::Trial::report) to record an intermediate value. +//! 2. Call [`Trial::should_prune`](crate::Trial::should_prune) to check the pruner's decision. +//! 3. If the pruner says prune, return [`TrialPruned`](crate::TrialPruned) from the objective. +//! +//! # Available pruners +//! +//! | Pruner | Algorithm | Best for | +//! |--------|-----------|----------| +//! | [`MedianPruner`] | Prune below median at each step | General-purpose default | +//! | [`PercentilePruner`] | Prune below configurable percentile | Tunable aggressiveness | +//! | [`ThresholdPruner`] | Prune outside fixed bounds | Known divergence limits | +//! | [`PatientPruner`] | Require N consecutive prune signals | Noisy intermediate values | +//! | [`SuccessiveHalvingPruner`] | Keep top 1/η fraction at each rung | Budget-aware pruning | +//! | [`HyperbandPruner`] | Multiple SHA brackets with different budgets | Robust to budget choice | +//! | [`WilcoxonPruner`] | Statistical signed-rank test vs. best trial | Rigorous noisy pruning | +//! | [`NopPruner`] | Never prune | Disabling pruning explicitly | +//! +//! # When to use pruning +//! +//! Pruning is most beneficial when: +//! +//! - The objective function has a natural notion of "steps" (e.g., training epochs) +//! - Early steps are informative about final performance +//! - Trials are expensive enough that stopping bad ones early saves significant time +//! +//! Start with [`MedianPruner`] for most use cases. Switch to [`WilcoxonPruner`] +//! if your intermediate values are noisy, or to [`HyperbandPruner`] if you want +//! automatic budget scheduling. mod hyperband; mod median; diff --git a/src/pruner/nop.rs b/src/pruner/nop.rs index 9ccfad9..ad9f337 100644 --- a/src/pruner/nop.rs +++ b/src/pruner/nop.rs @@ -1,3 +1,15 @@ +//! No-op pruner — never prune any trial. +//! +//! This is the default pruner used when no pruner is configured on a +//! [`Study`](crate::Study). It unconditionally returns `false` for every +//! pruning decision, allowing all trials to run to completion. +//! +//! # When to use +//! +//! - When you want to explicitly disable pruning +//! - As a baseline to compare against other pruners +//! - Already used by default — you rarely need to configure this manually + use super::Pruner; use crate::sampler::CompletedTrial; diff --git a/src/pruner/patient.rs b/src/pruner/patient.rs index 0f56f1c..ab94fd6 100644 --- a/src/pruner/patient.rs +++ b/src/pruner/patient.rs @@ -1,3 +1,34 @@ +//! Patient pruner — require consecutive prune signals before actually pruning. +//! +//! Wraps any other pruner and adds a patience window: the inner pruner +//! must recommend pruning for `patience` consecutive steps before the +//! trial is actually pruned. This prevents premature pruning when +//! intermediate values are noisy and a single bad step doesn't indicate +//! a truly bad trial. +//! +//! # When to use +//! +//! - When your intermediate values have high variance (e.g., mini-batch loss) +//! - When the inner pruner is too aggressive on its own +//! - To add robustness to any statistical pruner without changing its threshold +//! +//! # Configuration +//! +//! | Option | Default | Description | +//! |--------|---------|-------------| +//! | `inner` | *(required)* | The underlying pruner to wrap | +//! | `patience` | *(required)* | Number of consecutive prune signals required | +//! +//! # Example +//! +//! ``` +//! use optimizer::pruner::{PatientPruner, ThresholdPruner}; +//! +//! // Only prune after the threshold is exceeded 3 times in a row +//! let inner = ThresholdPruner::new().upper(100.0); +//! let pruner = PatientPruner::new(inner, 3); +//! ``` + use std::collections::HashMap; use std::sync::Mutex; diff --git a/src/pruner/percentile.rs b/src/pruner/percentile.rs index 7a5d6f6..0495b6b 100644 --- a/src/pruner/percentile.rs +++ b/src/pruner/percentile.rs @@ -1,3 +1,37 @@ +//! Percentile pruner — prune trials outside the top N% at each step. +//! +//! A generalization of [`MedianPruner`](super::MedianPruner) that lets you +//! control how aggressively to prune. At each step, the current trial's +//! intermediate value is compared against the given percentile of all +//! completed trials' values at the same step. +//! +//! # When to use +//! +//! - When you want finer control over pruning aggressiveness than median pruning +//! - Lower percentiles (e.g., 25%) are more aggressive — only keep the best quarter +//! - Higher percentiles (e.g., 75%) are more lenient — keep the top three quarters +//! - Percentile 50% is equivalent to [`MedianPruner`](super::MedianPruner) +//! +//! # Configuration +//! +//! | Option | Default | Description | +//! |--------|---------|-------------| +//! | `percentile` | *(required)* | Keep trials in the top N% — range `(0, 100)` | +//! | `n_warmup_steps` | 0 | Skip pruning in the first N steps | +//! | `n_min_trials` | 1 | Require at least N completed trials before pruning | +//! +//! # Example +//! +//! ``` +//! use optimizer::Direction; +//! use optimizer::pruner::PercentilePruner; +//! +//! // Keep only the top 25% of trials (aggressive pruning) +//! let pruner = PercentilePruner::new(25.0, Direction::Minimize) +//! .n_warmup_steps(5) +//! .n_min_trials(3); +//! ``` + use super::Pruner; use crate::sampler::CompletedTrial; use crate::types::{Direction, TrialState}; diff --git a/src/pruner/successive_halving.rs b/src/pruner/successive_halving.rs index 376ddea..0408101 100644 --- a/src/pruner/successive_halving.rs +++ b/src/pruner/successive_halving.rs @@ -1,3 +1,53 @@ +//! Successive Halving (SHA) pruner — budget-aware pruning at exponential rungs. +//! +//! Trials are evaluated at exponentially-spaced "rungs" (checkpoints). At each +//! rung, only the top 1/η fraction of trials survive to the next rung. This +//! is a principled way to allocate compute budget: give many trials a small +//! budget, then progressively invest more in the best ones. +//! +//! For example, with `min_resource=1`, `max_resource=81`, `reduction_factor=3`: +//! +//! | Rung | Step | Survivors | +//! |------|------|-----------| +//! | 0 | 1 | top 1/3 | +//! | 1 | 3 | top 1/3 | +//! | 2 | 9 | top 1/3 | +//! | 3 | 27 | top 1/3 | +//! | 4 | 81 | all (full budget) | +//! +//! # When to use +//! +//! - When your objective has a natural "budget" dimension (epochs, iterations) +//! - When early performance is a reasonable predictor of final performance +//! - When you want a principled alternative to median pruning +//! +//! If you're unsure about the right `min_resource`, consider +//! [`HyperbandPruner`](super::HyperbandPruner) which runs multiple brackets +//! to hedge against that choice. +//! +//! # Configuration +//! +//! | Option | Default | Description | +//! |--------|---------|-------------| +//! | `min_resource` | 1 | Step at which the first rung is placed | +//! | `max_resource` | 81 | Full budget (final rung, no pruning) | +//! | `reduction_factor` | 3 | At each rung, keep top 1/η trials | +//! | `min_early_stopping_rate` | 0 | Skip the first N rungs | +//! | `direction` | `Minimize` | Optimization direction | +//! +//! # Example +//! +//! ``` +//! use optimizer::Direction; +//! use optimizer::pruner::SuccessiveHalvingPruner; +//! +//! let pruner = SuccessiveHalvingPruner::new() +//! .min_resource(1) +//! .max_resource(81) +//! .reduction_factor(3) +//! .direction(Direction::Minimize); +//! ``` + use super::Pruner; use crate::sampler::CompletedTrial; use crate::types::{Direction, TrialState}; diff --git a/src/pruner/threshold.rs b/src/pruner/threshold.rs index 7058a71..837df9f 100644 --- a/src/pruner/threshold.rs +++ b/src/pruner/threshold.rs @@ -1,3 +1,33 @@ +//! Threshold pruner — prune trials whose values fall outside fixed bounds. +//! +//! Unlike statistical pruners that compare against other trials, the +//! threshold pruner uses absolute bounds. Any trial whose latest +//! intermediate value exceeds the upper bound or falls below the lower +//! bound is pruned immediately. +//! +//! # When to use +//! +//! - When you know hard limits for valid intermediate values (e.g., loss should +//! never exceed 100.0) +//! - To catch diverging or NaN-producing trials early +//! - Often combined with other pruners via [`PatientPruner`](super::PatientPruner) +//! +//! # Configuration +//! +//! | Option | Default | Description | +//! |--------|---------|-------------| +//! | `upper` | `None` | Prune if value exceeds this bound | +//! | `lower` | `None` | Prune if value falls below this bound | +//! +//! # Example +//! +//! ``` +//! use optimizer::pruner::ThresholdPruner; +//! +//! // Prune if loss exceeds 100.0 or accuracy drops below 0.0 +//! let pruner = ThresholdPruner::new().upper(100.0).lower(0.0); +//! ``` + use super::Pruner; use crate::sampler::CompletedTrial; diff --git a/src/pruner/wilcoxon.rs b/src/pruner/wilcoxon.rs index d148f20..9717e37 100644 --- a/src/pruner/wilcoxon.rs +++ b/src/pruner/wilcoxon.rs @@ -1,3 +1,44 @@ +//! Wilcoxon pruner — statistically rigorous pruning for noisy objectives. +//! +//! Uses the Wilcoxon signed-rank test to compare the current trial's +//! intermediate values against the best completed trial at matching steps. +//! The test accounts for the paired, step-aligned nature of the comparison +//! and only prunes when the difference is statistically significant. +//! +//! This is more principled than [`MedianPruner`](super::MedianPruner) for +//! noisy objectives because a single bad step won't trigger pruning — the +//! test considers the full distribution of paired differences. +//! +//! # When to use +//! +//! - When intermediate values have high variance (e.g., mini-batch loss, +//! stochastic reward signals) +//! - When you want a statistical guarantee that pruned trials are truly worse +//! - When you have enough steps (at least 6) for a meaningful test +//! +//! For less noisy objectives, [`MedianPruner`](super::MedianPruner) is simpler +//! and often sufficient. +//! +//! # Configuration +//! +//! | Option | Default | Description | +//! |--------|---------|-------------| +//! | `p_value_threshold` | 0.05 | Significance level — lower is more conservative | +//! | `n_warmup_steps` | 0 | Skip pruning in the first N steps | +//! | `n_min_trials` | 1 | Require at least N completed trials before pruning | +//! +//! # Example +//! +//! ``` +//! use optimizer::Direction; +//! use optimizer::pruner::WilcoxonPruner; +//! +//! let pruner = WilcoxonPruner::new(Direction::Minimize) +//! .p_value_threshold(0.05) +//! .n_warmup_steps(5) +//! .n_min_trials(1); +//! ``` + use core::cmp::Ordering; use super::Pruner; diff --git a/src/sampler/bohb.rs b/src/sampler/bohb.rs index 2444e8f..2e2d0b8 100644 --- a/src/sampler/bohb.rs +++ b/src/sampler/bohb.rs @@ -1,13 +1,13 @@ //! BOHB (Bayesian Optimization + `HyperBand`) sampler. //! -//! BOHB combines TPE's model-guided sampling with Hyperband's budget-aware +//! BOHB combines TPE's model-guided sampling with `HyperBand`'s budget-aware //! evaluation. Instead of building one global TPE model, BOHB conditions //! its TPE model on trials evaluated at a specific budget level, giving -//! better-calibrated proposals for each rung of the Hyperband schedule. +//! better-calibrated proposals for each rung of the `HyperBand` schedule. //! //! # How it works //! -//! 1. Compute all Hyperband rung steps (budget levels) from the config. +//! 1. Compute all `HyperBand` rung steps (budget levels) from the config. //! 2. On each `sample()` call, scan the history's `intermediate_values` //! to find the **largest budget level** with enough observations //! (`>= min_points_in_model`). @@ -16,6 +16,26 @@ //! 4. Delegate to an internal [`TpeSampler`] for the actual sampling. //! 5. Fall back to random sampling if no budget level has enough data. //! +//! # When to use +//! +//! - You are tuning hyperparameters for models that support early stopping +//! (e.g., neural networks with configurable epoch counts). +//! - You want to combine model-guided search with aggressive pruning of +//! unpromising configurations. +//! - Your objective has a natural "budget" axis (epochs, iterations, data +//! fraction) reported via [`Trial::report`](crate::Trial::report). +//! +//! Pair `BohbSampler` with [`matching_pruner`](BohbSampler::matching_pruner) +//! to get a `HyperBandPruner` whose budget schedule is consistent with +//! the sampler's conditioning levels. +//! +//! # Configuration +//! +//! - `min_resource` / `max_resource` — budget range (default: 1 … 81) +//! - `reduction_factor` (η) — successive halving factor (default: 3) +//! - `min_points_in_model` — observations needed before TPE replaces random (default: 10) +//! - All [`TpeSamplerBuilder`](super::tpe::TpeSamplerBuilder) options (gamma, seed, etc.) +//! //! # Examples //! //! ``` @@ -27,7 +47,7 @@ //! let study: Study<f64> = Study::with_sampler_and_pruner(Direction::Minimize, bohb, pruner); //! ``` //! -//! Using the builder for custom configuration: +//! Custom configuration via builder: //! //! ``` //! use optimizer::sampler::bohb::BohbSampler; @@ -50,7 +70,7 @@ use crate::sampler::tpe::TpeSampler; use crate::sampler::{CompletedTrial, Sampler}; use crate::types::Direction; -/// A BOHB sampler that combines TPE with Hyperband budget awareness. +/// A BOHB sampler that combines TPE with `HyperBand` budget awareness. /// /// BOHB filters trial history by budget level before delegating to TPE, /// so the surrogate model is conditioned on trials evaluated at the same @@ -58,7 +78,25 @@ use crate::types::Direction; /// than using a single global model across all budgets. /// /// Use [`BohbSampler::matching_pruner`] to create a [`HyperbandPruner`] -/// with matching parameters. +/// with matching `HyperBand` parameters. +/// +/// # Examples +/// +/// ``` +/// use optimizer::parameter::{FloatParam, Parameter}; +/// use optimizer::sampler::bohb::BohbSampler; +/// use optimizer::{Direction, Study}; +/// +/// let bohb = BohbSampler::builder() +/// .min_resource(1) +/// .max_resource(27) +/// .reduction_factor(3) +/// .seed(42) +/// .build() +/// .unwrap(); +/// let pruner = bohb.matching_pruner(Direction::Minimize); +/// let study: Study<f64> = Study::with_sampler_and_pruner(Direction::Minimize, bohb, pruner); +/// ``` pub struct BohbSampler { min_resource: u64, max_resource: u64, @@ -225,6 +263,14 @@ impl Sampler for BohbSampler { /// Builder for configuring a [`BohbSampler`]. /// +/// # Defaults +/// +/// - `min_resource`: 1 +/// - `max_resource`: 81 +/// - `reduction_factor`: 3 (η) +/// - `min_points_in_model`: 10 +/// - TPE: default settings (gamma = 0.25, etc.) +/// /// # Examples /// /// ``` diff --git a/src/sampler/cma_es.rs b/src/sampler/cma_es.rs index eadaec8..f243a88 100644 --- a/src/sampler/cma_es.rs +++ b/src/sampler/cma_es.rs @@ -1,15 +1,55 @@ //! CMA-ES (Covariance Matrix Adaptation Evolution Strategy) sampler. //! -//! CMA-ES maintains a multivariate Gaussian distribution over continuous -//! parameters and adapts its mean, covariance matrix, and step-size based -//! on trial rankings. It is one of the most effective derivative-free -//! optimizers for continuous search spaces. +//! CMA-ES is a stochastic, population-based optimizer that maintains a +//! multivariate Gaussian distribution over continuous parameters and adapts +//! its **mean**, **covariance matrix**, and **step-size** (σ) based on +//! trial rankings. It is widely regarded as one of the most effective +//! derivative-free optimizers for continuous search spaces. //! -//! Categorical parameters are sampled uniformly at random (not part of -//! the CMA-ES vector). If all parameters are categorical, the sampler -//! falls back to pure random sampling. +//! # Algorithm overview //! -//! Requires the `cma-es` feature flag. +//! Each generation: +//! 1. **Sample** λ (population size) candidates from N(m, σ²C). +//! 2. **Evaluate** and **rank** the candidates by objective value. +//! 3. **Update** the mean toward the best μ candidates (weighted recombination). +//! 4. **Adapt** the covariance matrix C via rank-one and rank-μ updates, and +//! adapt σ via cumulative step-size adaptation (CSA). +//! +//! Over time the search distribution narrows and rotates to align with the +//! landscape, efficiently exploiting structure in the objective function. +//! +//! # When to use +//! +//! - **Continuous parameters only** (float/int). Categorical parameters are +//! sampled uniformly at random and do not participate in the CMA-ES model. +//! - **Moderate dimensionality** — works well up to ~100 continuous dimensions. +//! Beyond that, the O(n²) covariance matrix becomes expensive to maintain. +//! - **Non-separable objectives** — CMA-ES learns parameter correlations +//! through the covariance matrix, making it especially effective on +//! rotated or ill-conditioned landscapes. +//! - **Moderate evaluation budgets** — typically needs ≈10×n to 100×n +//! evaluations to converge, where n is the number of continuous dimensions. +//! +//! For very cheap evaluations in low dimensions (d ≤ 20), consider +//! [`GpSampler`](super::gp::GpSampler) instead. For high-dimensional +//! separable problems, TPE may be more efficient. +//! +//! # Configuration +//! +//! | Option | Default | Description | +//! |--------|---------|-------------| +//! | `sigma0` | `avg_range / 4` | Initial step size — controls exploration breadth | +//! | `population_size` | `4 + ⌊3 ln n⌋` | Candidates per generation (λ) | +//! | `seed` | random | RNG seed for reproducibility | +//! +//! # Feature flag +//! +//! Requires the **`cma-es`** feature (adds the `nalgebra` dependency): +//! +//! ```toml +//! [dependencies] +//! optimizer = { version = "...", features = ["cma-es"] } +//! ``` //! //! # Examples //! @@ -17,8 +57,14 @@ //! use optimizer::sampler::cma_es::CmaEsSampler; //! use optimizer::{Direction, Study}; //! -//! let sampler = CmaEsSampler::with_seed(42); -//! let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler); +//! // Minimize a 2-D sphere function with CMA-ES +//! let sampler = CmaEsSampler::builder() +//! .sigma0(1.0) +//! .population_size(10) +//! .seed(42) +//! .build(); +//! +//! let mut study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler); //! ``` use std::collections::HashMap; diff --git a/src/sampler/differential_evolution.rs b/src/sampler/differential_evolution.rs index 0be8ffc..3a11e55 100644 --- a/src/sampler/differential_evolution.rs +++ b/src/sampler/differential_evolution.rs @@ -1,22 +1,66 @@ //! Differential Evolution (DE) sampler. //! -//! DE is a population-based metaheuristic that maintains a population of -//! candidate solutions and creates new candidates by combining (mutating + -//! crossing over) existing ones. It is competitive with CMA-ES on many -//! problems and simpler to implement. +//! DE is a population-based metaheuristic that maintains a pool of candidate +//! solutions and creates new candidates through **mutation** (combining +//! difference vectors of existing members) and **binomial crossover**. A +//! trial vector replaces its parent only if it achieves a better objective +//! value, guaranteeing monotonic improvement of the population. //! -//! Categorical parameters are sampled uniformly at random (not part of the -//! DE vector). If all parameters are categorical, the sampler falls back to -//! pure random sampling. +//! # Algorithm overview +//! +//! Each generation, for every population member *xᵢ*: +//! 1. **Mutation** — create a mutant vector *v* from other population +//! members using the selected [`DifferentialEvolutionStrategy`]: +//! - `Rand1`: `v = x_r1 + F * (x_r2 - x_r3)` +//! - `Best1`: `v = x_best + F * (x_r1 - x_r2)` +//! - `CurrentToBest1`: `v = x_i + F * (x_best - x_i) + F * (x_r1 - x_r2)` +//! 2. **Crossover** — create a trial vector *u* by mixing *v* and *xᵢ* +//! dimension-by-dimension with probability CR. +//! 3. **Selection** — replace *xᵢ* with *u* if `f(u) ≤ f(xᵢ)`. +//! +//! # When to use +//! +//! - **Continuous parameters** (float/int). Categorical parameters are +//! sampled uniformly at random and do not participate in DE. +//! - **Moderate to large search spaces** — DE scales better than GP-based +//! methods to higher dimensions, though it may need more evaluations. +//! - **Multi-modal landscapes** — the `Rand1` strategy maintains diversity +//! and avoids premature convergence. +//! - **No feature flags required** — DE is available with default features. +//! +//! For non-separable problems in moderate dimensions, consider +//! [`CmaEsSampler`](super::cma_es::CmaEsSampler) which learns parameter +//! correlations. For expensive functions with few dimensions, consider +//! [`GpSampler`](super::gp::GpSampler). +//! +//! # Configuration +//! +//! | Option | Default | Description | +//! |--------|---------|-------------| +//! | `population_size` | `max(10n, 15)` | Candidates per generation | +//! | `mutation_factor` (F) | 0.8 | Differential amplification — higher = more exploration | +//! | `crossover_rate` (CR) | 0.9 | Probability of taking a dimension from the mutant | +//! | `strategy` | `Rand1` | Mutation strategy (see [`DifferentialEvolutionStrategy`]) | +//! | `seed` | random | RNG seed for reproducibility | //! //! # Examples //! //! ``` -//! use optimizer::sampler::differential_evolution::DifferentialEvolutionSampler; +//! use optimizer::sampler::differential_evolution::{ +//! DifferentialEvolutionSampler, DifferentialEvolutionStrategy, +//! }; //! use optimizer::{Direction, Study}; //! -//! let sampler = DifferentialEvolutionSampler::with_seed(42); -//! let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler); +//! // Minimize with DE using the Best1 strategy for faster convergence +//! let sampler = DifferentialEvolutionSampler::builder() +//! .mutation_factor(0.7) +//! .crossover_rate(0.9) +//! .strategy(DifferentialEvolutionStrategy::Best1) +//! .population_size(20) +//! .seed(42) +//! .build(); +//! +//! let mut study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler); //! ``` use std::collections::HashMap; diff --git a/src/sampler/gp.rs b/src/sampler/gp.rs index 2ae49b1..fe5d8f6 100644 --- a/src/sampler/gp.rs +++ b/src/sampler/gp.rs @@ -1,15 +1,60 @@ //! Gaussian Process (GP) sampler with Expected Improvement acquisition. //! -//! A classical Bayesian optimization sampler that uses a Gaussian Process -//! surrogate model with a Matérn 5/2 kernel and Expected Improvement (EI) +//! A classical Bayesian optimization sampler that builds a Gaussian Process +//! surrogate model with a **Matérn 5/2 kernel** (with ARD lengthscales) and +//! selects the next trial by maximizing the **Expected Improvement (EI)** //! acquisition function. Best suited for small, expensive evaluations in //! low-dimensional continuous spaces (d ≤ 20). //! -//! Categorical parameters are sampled uniformly at random (not part of -//! the GP model). If all parameters are categorical, the sampler falls -//! back to pure random sampling. +//! # Algorithm overview //! -//! Requires the `gp` feature flag. +//! 1. **Startup phase** — the first `n_startup_trials` trials are sampled +//! uniformly at random to build an initial dataset. +//! 2. **Fit GP** — training observations are standardized (zero mean, unit +//! variance) and a GP with Matérn 5/2 kernel is fitted via Cholesky +//! decomposition. ARD lengthscales are set to the per-dimension standard +//! deviation of the training inputs. +//! 3. **Maximize EI** — `n_candidates` random points are evaluated under +//! the GP posterior and the point with the highest Expected Improvement +//! is returned as the next trial. +//! +//! The GP uses at most 100 training points (the most recent ones) to keep +//! the O(n³) fitting cost manageable. +//! +//! # When to use +//! +//! - **Expensive objective functions** where every evaluation is costly +//! (e.g. physical experiments, large simulations). The GP surrogate +//! amortizes this cost by making fewer evaluations. +//! - **Low-dimensional continuous spaces** — typically d ≤ 20. Beyond that, +//! the GP becomes unreliable and alternatives like +//! [`CmaEsSampler`](super::cma_es::CmaEsSampler) or +//! [`TpeSampler`](super::tpe::TpeSampler) are preferable. +//! - **Smooth, low-noise objectives** — the GP assumes smoothness through +//! the Matérn 5/2 kernel. Very noisy objectives require increasing +//! `noise_variance`. +//! +//! Categorical parameters are sampled uniformly at random and do not +//! participate in the GP model. If all parameters are categorical, the +//! sampler falls back to pure random sampling. +//! +//! # Configuration +//! +//! | Option | Default | Description | +//! |--------|---------|-------------| +//! | `n_startup_trials` | 10 | Random trials before GP-guided sampling begins | +//! | `n_candidates` | 1000 | Random candidates for EI maximization | +//! | `noise_variance` | 1e-6 | Observation noise added to kernel diagonal | +//! | `seed` | random | RNG seed for reproducibility | +//! +//! # Feature flag +//! +//! Requires the **`gp`** feature (adds the `nalgebra` dependency): +//! +//! ```toml +//! [dependencies] +//! optimizer = { version = "...", features = ["gp"] } +//! ``` //! //! # Examples //! @@ -17,8 +62,14 @@ //! use optimizer::sampler::gp::GpSampler; //! use optimizer::{Direction, Study}; //! -//! let sampler = GpSampler::with_seed(42); -//! let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler); +//! // Minimize an expensive function with GP-based Bayesian optimization +//! let sampler = GpSampler::builder() +//! .n_startup_trials(5) +//! .n_candidates(500) +//! .seed(42) +//! .build(); +//! +//! let mut study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler); //! ``` use std::collections::HashMap; diff --git a/src/sampler/grid.rs b/src/sampler/grid.rs index dd6768f..c3ce2b7 100644 --- a/src/sampler/grid.rs +++ b/src/sampler/grid.rs @@ -1,7 +1,39 @@ -//! Grid search sampler implementation. +//! Grid search sampler — exhaustive evaluation of discretized parameter spaces. //! -//! `GridSearchSampler` performs exhaustive grid search over the parameter space, -//! systematically evaluating all combinations of discretized parameter values. +//! [`GridSearchSampler`] divides each parameter range into a fixed number of +//! evenly spaced points (or uses the explicit step size when defined) and +//! evaluates them sequentially. This guarantees complete coverage of the +//! search grid at the cost of scaling exponentially with the number of +//! parameters. +//! +//! # When to use +//! +//! - **Small, discrete spaces** — when you have a handful of categorical or +//! integer parameters and want to evaluate every combination. +//! - **Reproducibility** — grid search is fully deterministic with no random +//! component. +//! - **Benchmarking** — compare grid search results against adaptive samplers +//! to measure their benefit. +//! +//! Avoid grid search for high-dimensional or large continuous spaces; +//! prefer [`TpeSampler`](super::tpe::TpeSampler) or +//! [`RandomSampler`](super::random::RandomSampler) instead. +//! +//! # Configuration +//! +//! | Option | Default | Description | +//! |---|---|---| +//! | `n_points_per_param` | 10 | Points per continuous parameter (ignored when `step` is set) | +//! +//! # Example +//! +//! ``` +//! use optimizer::prelude::*; +//! use optimizer::sampler::grid::GridSearchSampler; +//! +//! let sampler = GridSearchSampler::builder().n_points_per_param(5).build(); +//! let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler); +//! ``` use std::collections::HashMap; @@ -295,38 +327,38 @@ struct GridState { grids: HashMap<String, CachedGrid>, } -/// A grid search sampler that exhaustively evaluates all grid points. +/// Exhaustive grid search sampler. /// -/// `GridSearchSampler` divides the parameter space into a grid and systematically -/// samples each point. This is useful when you want to evaluate all combinations -/// of parameter values, especially for discrete or small parameter spaces. +/// Divide each parameter range into evenly spaced points and evaluate them +/// sequentially. When a parameter has an explicit `step` size, the step grid +/// is used instead of auto-discretization. /// -/// # Grid Exhaustion +/// Grid state is tracked **per distribution key** (bounds + step + log-scale). +/// Parameters with identical distributions share the same grid counter, so +/// use distinct ranges when multiple parameters span the same domain. /// -/// The sampler tracks its position in the grid for each distribution independently. -/// When all grid points for a distribution have been sampled, subsequent calls to -/// `sample()` for that distribution will **panic** with the message: -/// `"GridSearchSampler: all grid points exhausted"`. +/// # Grid exhaustion /// -/// To avoid panics, use [`is_exhausted()`](Self::is_exhausted) to check if all -/// points have been sampled before calling `sample()`. You can also use -/// [`grid_size()`](Self::grid_size) to determine the total number of grid points -/// that will be sampled. +/// When all grid points for a distribution have been sampled, the next +/// `sample()` call for that distribution **panics**. Use +/// [`is_exhausted()`](Self::is_exhausted) to check before sampling, or +/// set `n_points_per_param` high enough to cover the planned number of +/// trials. /// -/// # Thread Safety +/// # Thread safety /// -/// `GridSearchSampler` is thread-safe (`Send + Sync`) and uses internal locking -/// to ensure safe concurrent access to grid state. +/// `GridSearchSampler` is `Send + Sync` and uses internal locking for +/// safe concurrent access. /// /// # Examples /// /// ``` /// use optimizer::sampler::grid::GridSearchSampler; /// -/// // Create with default settings (10 points per parameter) +/// // Default: 10 points per parameter /// let sampler = GridSearchSampler::new(); /// -/// // Create with custom settings using the builder +/// // Custom grid density /// let sampler = GridSearchSampler::builder().n_points_per_param(20).build(); /// ``` pub struct GridSearchSampler { diff --git a/src/sampler/moead.rs b/src/sampler/moead.rs index d6d3997..6f11d44 100644 --- a/src/sampler/moead.rs +++ b/src/sampler/moead.rs @@ -1,8 +1,58 @@ //! MOEA/D (Multi-Objective Evolutionary Algorithm based on Decomposition) sampler. //! -//! Decomposes a multi-objective problem into scalar subproblems using -//! weight vectors and solves them collaboratively. Supports Weighted Sum, -//! Tchebycheff, and Penalty-based Boundary Intersection (PBI) scalarization. +//! MOEA/D takes a fundamentally different approach from Pareto-based +//! algorithms like NSGA-II/III. It **decomposes** the multi-objective +//! problem into a set of scalar subproblems using evenly distributed +//! weight vectors (Das-Dennis points), then solves them collaboratively +//! through **neighborhood-based mating and replacement**. +//! +//! # Algorithm +//! +//! 1. **Decompose** — generate weight vectors on the unit simplex and +//! assign one scalar subproblem per weight vector. +//! 2. **Build neighborhoods** — for each subproblem, find its T nearest +//! neighbors by Euclidean distance between weight vectors. +//! 3. **Mate from neighborhood** — select parents from the neighborhood +//! of each subproblem and produce offspring via SBX crossover + +//! polynomial mutation. +//! 4. **Scalarize and update** — evaluate offspring using a scalarization +//! function and update neighboring subproblems if the offspring improves +//! their scalar value. +//! 5. **Update ideal point** — track the best value seen per objective. +//! +//! # Scalarization methods +//! +//! | Method | Formula | Best for | +//! |--------|---------|----------| +//! | [`Tchebycheff`](Decomposition::Tchebycheff) (default) | `max(wᵢ * \|fᵢ - zᵢ*\|)` | General purpose, handles non-convex fronts | +//! | [`WeightedSum`](Decomposition::WeightedSum) | `Σ(wᵢ * fᵢ)` | Convex Pareto fronts only | +//! | [`Pbi`](Decomposition::Pbi) | `d₁ + θ * d₂` | Fine-grained convergence/diversity control | +//! +//! # When to use +//! +//! - Problems where you want **evenly distributed** solutions along the +//! Pareto front (one solution per weight direction). +//! - Many-objective optimization (3+ objectives) — scales well because +//! each subproblem is a simple scalar optimization. +//! - Problems with **non-convex** Pareto fronts (use Tchebycheff or PBI). +//! - When you need explicit control over the trade-off distribution via +//! weight vectors. +//! +//! For Pareto-based approaches, see +//! [`Nsga2Sampler`](super::nsga2::Nsga2Sampler) (crowding distance) or +//! [`Nsga3Sampler`](super::nsga3::Nsga3Sampler) (reference-point niching). +//! +//! # Configuration +//! +//! | Parameter | Builder method | Default | +//! |-----------|---------------|---------| +//! | Population size | [`population_size`](MoeadSamplerBuilder::population_size) | Number of Das-Dennis weight vectors | +//! | Neighborhood size (T) | [`neighborhood_size`](MoeadSamplerBuilder::neighborhood_size) | `min(20, pop_size)` | +//! | Decomposition method | [`decomposition`](MoeadSamplerBuilder::decomposition) | Tchebycheff | +//! | Crossover probability | [`crossover_prob`](MoeadSamplerBuilder::crossover_prob) | 1.0 | +//! | SBX distribution index | [`crossover_eta`](MoeadSamplerBuilder::crossover_eta) | 20.0 | +//! | Mutation distribution index | [`mutation_eta`](MoeadSamplerBuilder::mutation_eta) | 20.0 | +//! | Random seed | [`seed`](MoeadSamplerBuilder::seed) | random | //! //! # Examples //! @@ -37,15 +87,29 @@ use crate::multi_objective::MultiObjectiveTrial; use crate::param::ParamValue; use crate::types::Direction; -/// Decomposition (scalarization) method for MOEA/D. +/// Decomposition (scalarization) method for [`MoeadSampler`]. +/// +/// Control how multi-objective values are reduced to a single scalar +/// for each subproblem. The default is [`Tchebycheff`](Self::Tchebycheff), +/// which handles both convex and non-convex Pareto fronts. #[derive(Debug, Clone, Default)] pub enum Decomposition { - /// Weighted sum: `sum(w_i * f_i)`. + /// Weighted sum: `Σ(wᵢ * fᵢ)`. + /// + /// Simplest method but can only find solutions on convex regions + /// of the Pareto front. WeightedSum, - /// Tchebycheff: `max(w_i * |f_i - z_i*|)`. + /// Tchebycheff: `max(wᵢ * |fᵢ - zᵢ*|)`. + /// + /// Handles non-convex Pareto fronts. The most commonly used + /// decomposition method (default). #[default] Tchebycheff, - /// Penalty-based Boundary Intersection with parameter theta. + /// Penalty-based Boundary Intersection: `d₁ + θ * d₂`. + /// + /// Provides fine-grained control over the convergence/diversity + /// balance via the penalty parameter `theta`. Higher `theta` + /// favors solutions closer to the weight direction. Pbi { /// Penalty parameter controlling the balance between convergence /// and diversity. Default: 5.0. @@ -55,9 +119,14 @@ pub enum Decomposition { /// MOEA/D sampler for multi-objective optimization. /// -/// Decomposes the multi-objective problem into scalar subproblems -/// using weight vectors, solving them collaboratively via -/// neighborhood-based mating and replacement. +/// Decompose a multi-objective problem into scalar subproblems using +/// weight vectors and solve them collaboratively via neighborhood-based +/// mating. Supports [`Tchebycheff`](Decomposition::Tchebycheff), +/// [`WeightedSum`](Decomposition::WeightedSum), and +/// [`Pbi`](Decomposition::Pbi) scalarization. +/// +/// Create with [`MoeadSampler::new`], [`MoeadSampler::with_seed`], or +/// [`MoeadSampler::builder`] for full configuration. pub struct MoeadSampler { state: Mutex<MoeadState>, } diff --git a/src/sampler/motpe.rs b/src/sampler/motpe.rs index 4525218..823b791 100644 --- a/src/sampler/motpe.rs +++ b/src/sampler/motpe.rs @@ -1,19 +1,38 @@ //! Multi-Objective Tree-Parzen Estimator (MOTPE) sampler. //! -//! Extends TPE to handle multi-objective optimization by using Pareto -//! non-dominated sorting to define "good" vs "bad" trial regions for -//! the KDE models, replacing the single-objective gamma-based split. +//! MOTPE extends TPE to multi-objective optimization by replacing the gamma-based +//! split with Pareto non-dominated sorting. This lets the sampler propose +//! parameters that push the Pareto front forward across all objectives +//! simultaneously. //! //! # Algorithm //! -//! In single-objective TPE, trials are sorted by value and split at a -//! gamma percentile into good/bad groups. MOTPE replaces this with: +//! In single-objective TPE, trials are sorted by value and split at a gamma +//! percentile into good/bad groups. MOTPE replaces this with: //! -//! 1. Compute non-dominated sorting on all completed trials -//! 2. Use the Pareto front (rank 0) as "good" trials -//! 3. Use dominated trials as "bad" trials -//! 4. Build KDE l(x) from good, g(x) from bad -//! 5. Sample candidates and score by l(x)/g(x) +//! 1. Compute non-dominated sorting on all completed trials. +//! 2. Use the Pareto front (rank 0) as "good" trials. +//! 3. Use dominated trials (rank 1+) as "bad" trials. +//! 4. Build KDE l(x) from good, g(x) from bad. +//! 5. Sample candidates and score by l(x)/g(x). +//! +//! # When to use +//! +//! - You have 2+ objectives and want model-guided search (not pure evolutionary). +//! - Your objectives are relatively smooth and continuous. +//! - You want a Pareto-aware version of TPE without the overhead of full +//! population-based algorithms like NSGA-II or NSGA-III. +//! +//! For single-objective problems, use [`TpeSampler`](super::tpe::TpeSampler) instead. +//! For many-objective (3+) problems with reference-point decomposition, consider +//! NSGA-III or MOEA/D. +//! +//! # Configuration +//! +//! - `n_startup_trials` — number of random trials before MOTPE kicks in (default: 11) +//! - `n_ei_candidates` — candidates evaluated per sample (default: 24) +//! - `kde_bandwidth` — optional fixed KDE bandwidth; `None` uses Scott's rule +//! - `seed` — optional seed for reproducibility //! //! # Examples //! @@ -50,14 +69,21 @@ use crate::{pareto, rng_util}; /// Multi-Objective TPE (MOTPE) sampler for multi-objective Bayesian optimization. /// -/// Uses Pareto non-dominated sorting to split completed trials into -/// "good" (non-dominated, rank 0) and "bad" (dominated) groups, then -/// fits kernel density estimators to each group and samples new points -/// that maximize l(x)/g(x). +/// Use Pareto non-dominated sorting to split completed trials into "good" +/// (non-dominated, rank 0) and "bad" (dominated) groups, then fit kernel +/// density estimators to each group and sample new points that maximize +/// l(x)/g(x). /// /// During the startup phase (fewer than `n_startup_trials` completed), /// MOTPE falls back to random sampling. /// +/// # When to use +/// +/// Use `MotpeSampler` when optimizing 2+ objectives and you want a +/// model-guided sampler that adapts proposals based on the current +/// Pareto front. For single-objective problems, use +/// [`TpeSampler`](super::tpe::TpeSampler) instead. +/// /// # Examples /// /// ``` @@ -74,6 +100,17 @@ use crate::{pareto, rng_util}; /// /// let study = /// MultiObjectiveStudy::with_sampler(vec![Direction::Minimize, Direction::Minimize], sampler); +/// +/// let x = FloatParam::new(0.0, 1.0); +/// study +/// .optimize(30, |trial| { +/// let xv = x.suggest(trial)?; +/// Ok::<_, optimizer::Error>(vec![xv, 1.0 - xv]) +/// }) +/// .unwrap(); +/// +/// let front = study.pareto_front(); +/// assert!(!front.is_empty()); /// ``` pub struct MotpeSampler { /// Number of trials before MOTPE kicks in (uses random sampling before this). @@ -520,6 +557,13 @@ impl MultiObjectiveSampler for MotpeSampler { /// Builder for configuring a [`MotpeSampler`]. /// +/// # Defaults +/// +/// - `n_startup_trials`: 11 +/// - `n_ei_candidates`: 24 +/// - `kde_bandwidth`: None (Scott's rule) +/// - `seed`: None (OS entropy) +/// /// # Examples /// /// ``` diff --git a/src/sampler/nsga2.rs b/src/sampler/nsga2.rs index acbbab5..0a8f873 100644 --- a/src/sampler/nsga2.rs +++ b/src/sampler/nsga2.rs @@ -1,7 +1,45 @@ //! NSGA-II (Non-dominated Sorting Genetic Algorithm II) sampler. //! -//! Implements multi-objective optimization using non-dominated sorting, -//! crowding distance, SBX crossover, and polynomial mutation. +//! NSGA-II is one of the most widely used evolutionary multi-objective +//! optimization algorithms. It ranks the population using **non-dominated +//! sorting** (fast O(MN²) algorithm) and breaks ties within the same +//! Pareto front using **crowding distance**, which favors solutions in +//! less-crowded regions of the objective space. +//! +//! # Algorithm +//! +//! Each generation proceeds as follows: +//! +//! 1. **Non-dominated sorting** — partition the combined parent+offspring +//! population into Pareto fronts F₁, F₂, … +//! 2. **Crowding distance** — for each front, compute per-solution crowding +//! distance (sum of normalized neighbor gaps in each objective). +//! 3. **Selection** — fill the next population front-by-front. When a front +//! only partially fits, prefer solutions with higher crowding distance. +//! 4. **Binary tournament** — select parents using (rank, crowding distance) +//! comparisons. +//! 5. **SBX crossover + polynomial mutation** — generate offspring. +//! +//! # When to use +//! +//! - Two-objective problems where you want a well-spread Pareto front. +//! - General-purpose multi-objective optimization with moderate population +//! sizes. +//! - Problems that benefit from diversity preservation via crowding distance. +//! +//! For problems with **three or more objectives**, consider +//! [`Nsga3Sampler`](super::nsga3::Nsga3Sampler) (reference-point niching) +//! or [`MoeadSampler`](super::moead::MoeadSampler) (decomposition). +//! +//! # Configuration +//! +//! | Parameter | Builder method | Default | +//! |-----------|---------------|---------| +//! | Population size | [`population_size`](Nsga2SamplerBuilder::population_size) | `4 + floor(3 * ln(n_params))`, min 4 | +//! | Crossover probability | [`crossover_prob`](Nsga2SamplerBuilder::crossover_prob) | 0.9 | +//! | SBX distribution index | [`crossover_eta`](Nsga2SamplerBuilder::crossover_eta) | 20.0 | +//! | Mutation distribution index | [`mutation_eta`](Nsga2SamplerBuilder::mutation_eta) | 20.0 | +//! | Random seed | [`seed`](Nsga2SamplerBuilder::seed) | random | //! //! # Examples //! @@ -39,8 +77,12 @@ use crate::types::Direction; /// NSGA-II sampler for multi-objective optimization. /// -/// Provides non-dominated sorting, crowding distance selection, -/// SBX crossover, and polynomial mutation. +/// Use non-dominated sorting with crowding-distance tie-breaking to +/// evolve a well-spread Pareto front. Best suited for bi-objective +/// problems; for 3+ objectives prefer [`Nsga3Sampler`](super::nsga3::Nsga3Sampler). +/// +/// Create with [`Nsga2Sampler::new`], [`Nsga2Sampler::with_seed`], or +/// [`Nsga2Sampler::builder`] for full configuration. pub struct Nsga2Sampler { state: Mutex<Nsga2State>, } diff --git a/src/sampler/nsga3.rs b/src/sampler/nsga3.rs index 09f414c..44028ef 100644 --- a/src/sampler/nsga3.rs +++ b/src/sampler/nsga3.rs @@ -1,9 +1,49 @@ //! NSGA-III (Non-dominated Sorting Genetic Algorithm III) sampler. //! -//! Uses reference-point-based niching for better diversity in -//! many-objective (3+) optimization problems. Das-Dennis structured -//! reference points guide the search toward a well-distributed -//! Pareto front. +//! NSGA-III extends NSGA-II to handle **many-objective** (3+) problems +//! where crowding distance loses effectiveness. Instead of crowding +//! distance, it uses **reference-point-based niching** with structured +//! Das-Dennis reference points distributed on the unit simplex to guide +//! the population toward a well-diversified Pareto front. +//! +//! # Algorithm +//! +//! Each generation proceeds as follows: +//! +//! 1. **Non-dominated sorting** — same as NSGA-II, partition the +//! combined population into Pareto fronts F₁, F₂, … +//! 2. **Normalize objectives** — translate by ideal point and scale by +//! intercepts so all objectives lie in roughly \[0, 1\]. +//! 3. **Associate with reference points** — assign each solution to the +//! closest Das-Dennis reference direction by perpendicular distance. +//! 4. **Niching selection** — when the last front only partially fits, +//! prefer solutions associated with under-represented reference points +//! (lowest niche count first, closest distance second). +//! 5. **SBX crossover + polynomial mutation** — generate offspring via +//! rank-based tournament selection. +//! +//! # When to use +//! +//! - **Three or more objectives** — NSGA-III maintains diversity far +//! better than NSGA-II as the number of objectives grows. +//! - Problems where you want a **uniformly distributed** Pareto front +//! guided by structured reference points. +//! - Scales well up to ~10 objectives with appropriate division settings. +//! +//! For bi-objective problems, [`Nsga2Sampler`](super::nsga2::Nsga2Sampler) +//! is simpler and equally effective. For decomposition-based optimization, +//! see [`MoeadSampler`](super::moead::MoeadSampler). +//! +//! # Configuration +//! +//! | Parameter | Builder method | Default | +//! |-----------|---------------|---------| +//! | Population size | [`population_size`](Nsga3SamplerBuilder::population_size) | Number of Das-Dennis reference points | +//! | Das-Dennis divisions (H) | [`n_divisions`](Nsga3SamplerBuilder::n_divisions) | Auto-chosen from population size and objectives | +//! | Crossover probability | [`crossover_prob`](Nsga3SamplerBuilder::crossover_prob) | 1.0 | +//! | SBX distribution index | [`crossover_eta`](Nsga3SamplerBuilder::crossover_eta) | 30.0 | +//! | Mutation distribution index | [`mutation_eta`](Nsga3SamplerBuilder::mutation_eta) | 20.0 | +//! | Random seed | [`seed`](Nsga3SamplerBuilder::seed) | random | //! //! # Examples //! @@ -49,8 +89,12 @@ use crate::types::Direction; /// NSGA-III sampler for multi-objective optimization. /// -/// Uses reference-point-based niching to maintain diversity, -/// especially effective for problems with 3 or more objectives. +/// Use reference-point niching with Das-Dennis structured points to +/// maintain diversity in many-objective (3+) problems. For bi-objective +/// problems, [`Nsga2Sampler`](super::nsga2::Nsga2Sampler) is simpler. +/// +/// Create with [`Nsga3Sampler::new`], [`Nsga3Sampler::with_seed`], or +/// [`Nsga3Sampler::builder`] for full configuration. pub struct Nsga3Sampler { state: Mutex<Nsga3State>, } diff --git a/src/sampler/random.rs b/src/sampler/random.rs index e033a4e..292fe4a 100644 --- a/src/sampler/random.rs +++ b/src/sampler/random.rs @@ -1,4 +1,31 @@ -//! Random sampler implementation. +//! Random sampler — uniform independent sampling. +//! +//! [`RandomSampler`] draws each parameter value independently and uniformly +//! at random, ignoring trial history entirely. It respects log-scale and +//! step-size constraints defined by the parameter distribution. +//! +//! # When to use +//! +//! - **Baseline comparison** — run Random alongside smarter samplers to +//! quantify their benefit. +//! - **Startup phase** — many model-based samplers (TPE, GP, CMA-ES) use +//! random sampling for their first *n* trials before fitting a surrogate. +//! - **Very high dimensions** — when the search space is too large for +//! structured exploration, random search with enough budget can be +//! surprisingly competitive. +//! +//! For better uniform coverage without model fitting, consider +//! [`SobolSampler`](super::sobol::SobolSampler) (requires the `sobol` +//! feature flag). +//! +//! # Example +//! +//! ``` +//! use optimizer::prelude::*; +//! use optimizer::sampler::random::RandomSampler; +//! +//! let study: Study<f64> = Study::with_sampler(Direction::Minimize, RandomSampler::with_seed(42)); +//! ``` use parking_lot::Mutex; @@ -7,11 +34,15 @@ use crate::param::ParamValue; use crate::rng_util; use crate::sampler::{CompletedTrial, Sampler}; -/// A simple random sampler that samples uniformly from distributions. +/// Uniform independent random sampler. /// -/// This sampler ignores the trial history and samples uniformly at random, -/// respecting log scale and step size constraints. It serves as a baseline -/// sampler and is used during the startup phase of more sophisticated samplers. +/// Sample each parameter value uniformly at random, respecting log-scale and +/// step-size constraints. Trial history is ignored — every sample is drawn +/// independently. +/// +/// This is the default sampler used by [`Study::new`](crate::Study::new) +/// and during the startup phase of model-based samplers such as +/// [`TpeSampler`](super::tpe::TpeSampler). /// /// # Examples /// diff --git a/src/sampler/sobol.rs b/src/sampler/sobol.rs index 3c4bb18..2dad0b8 100644 --- a/src/sampler/sobol.rs +++ b/src/sampler/sobol.rs @@ -1,4 +1,48 @@ //! Quasi-random sampler using Sobol low-discrepancy sequences. +//! +//! [`SobolSampler`] generates points from a Sobol sequence (scrambled via the +//! Burley 2020 algorithm) to fill the parameter space more uniformly than +//! pure random sampling. Where [`RandomSampler`](super::random::RandomSampler) +//! may cluster points in some regions by chance, Sobol sequences are +//! constructed to spread points evenly across all dimensions. +//! +//! # When to use +//! +//! - **Better-than-random baseline** — when you want uniform coverage +//! without the cost of model fitting (TPE, GP, etc.). +//! - **Startup phase replacement** — use Sobol instead of random for the +//! initial exploration phase of adaptive samplers. +//! - **Moderate dimensionality** — Sobol uniformity is strongest up to +//! ~20 dimensions; beyond that the advantage over random sampling +//! diminishes. +//! - **Deterministic exploration** — Sobol sequences are fully deterministic +//! for a given seed, making experiments reproducible. +//! +//! # How it works +//! +//! Each trial maps to a Sobol sequence index, and each parameter within a +//! trial maps to a separate Sobol dimension. The resulting quasi-random +//! point in \[0, 1) is then scaled to the parameter's distribution (linear, +//! log-scale, or step grid). +//! +//! **Important:** parameters must be suggested in the same order across +//! trials for consistent dimension assignment. +//! +//! Requires the **`sobol`** feature flag: +//! +//! ```toml +//! [dependencies] +//! optimizer = { version = "...", features = ["sobol"] } +//! ``` +//! +//! # Example +//! +//! ``` +//! use optimizer::prelude::*; +//! use optimizer::sampler::sobol::SobolSampler; +//! +//! let study: Study<f64> = Study::with_sampler(Direction::Minimize, SobolSampler::with_seed(42)); +//! ``` use parking_lot::Mutex; use sobol_burley::sample; @@ -17,13 +61,11 @@ struct SobolState { /// Quasi-random sampler using Sobol low-discrepancy sequences. /// -/// Provides better uniform coverage of the parameter space than -/// [`RandomSampler`](super::random::RandomSampler). Useful as a baseline or -/// for the startup phase of model-based samplers. -/// -/// Unlike random sampling, Sobol sequences are deterministic and fill the -/// space more evenly, reducing the number of trials needed to adequately -/// cover the search space. +/// Produce better uniform coverage of the parameter space than +/// [`RandomSampler`](super::random::RandomSampler) by using a +/// scrambled Sobol sequence (Burley 2020). Useful as a standalone +/// baseline or as a drop-in replacement for the random startup +/// phase of model-based samplers. /// /// Each trial uses a different Sobol sequence index, and each parameter /// within a trial maps to a different Sobol dimension. Parameters must be @@ -33,7 +75,7 @@ struct SobolState { /// Sobol sequences are most effective in moderate dimensions (up to ~20). /// For very high dimensions, the uniformity advantage diminishes. /// -/// Requires the `sobol` feature flag. +/// Requires the **`sobol`** feature flag. /// /// # Examples /// diff --git a/src/sampler/tpe/mod.rs b/src/sampler/tpe/mod.rs index 3013111..2c17e3d 100644 --- a/src/sampler/tpe/mod.rs +++ b/src/sampler/tpe/mod.rs @@ -1,7 +1,63 @@ -//! Tree-Parzen Estimator (TPE) sampler implementation and utilities. +//! Tree-Parzen Estimator (TPE) sampler family for Bayesian optimization. //! -//! This module provides TPE-based sampling for Bayesian optimization, -//! including support for intersection search space calculation. +//! TPE is a sequential model-based optimization algorithm that models P(x|y) instead +//! of P(y|x). It splits completed trials into "good" (below the gamma quantile) and +//! "bad" groups, fits a kernel density estimator (KDE) to each, and proposes new +//! points by maximizing the l(x)/g(x) ratio — an approximation of Expected Improvement. +//! +//! # Samplers +//! +//! | Sampler | Models parameters | Best for | +//! |---------|-------------------|----------| +//! | [`TpeSampler`] | Independently | General-purpose single-objective optimization | +//! | [`MultivariateTpeSampler`] | Jointly | Problems with correlated parameters | +//! +//! # Gamma strategies +//! +//! The gamma quantile controls how many trials are considered "good". This module +//! provides four built-in strategies via the [`GammaStrategy`] trait: +//! +//! | Strategy | Formula | Default | +//! |----------|---------|---------| +//! | [`FixedGamma`] | Constant value | gamma = 0.25 | +//! | [`LinearGamma`] | Linear ramp from min to max | 0.10 → 0.25 over 100 trials | +//! | [`SqrtGamma`] | 1/√n decay (Optuna-style) | factor = 1.0, max = 0.25 | +//! | [`HyperoptGamma`] | (base+1)/n (Hyperopt-style) | base = 24, max = 0.25 | +//! +//! You can also implement [`GammaStrategy`] for a custom splitting rule. +//! +//! # Search-space utilities +//! +//! The [`search_space`] submodule provides [`IntersectionSearchSpace`] for computing +//! the common parameter set across trials, and [`GroupDecomposedSearchSpace`] for +//! splitting parameters into independent groups based on co-occurrence. +//! +//! # Examples +//! +//! Basic TPE with default settings: +//! +//! ``` +//! use optimizer::sampler::tpe::TpeSampler; +//! use optimizer::{Direction, Study}; +//! +//! let study: Study<f64> = Study::with_sampler(Direction::Minimize, TpeSampler::new()); +//! ``` +//! +//! Multivariate TPE for correlated parameters: +//! +//! ``` +//! use optimizer::sampler::tpe::MultivariateTpeSampler; +//! use optimizer::{Direction, Study}; +//! +//! let sampler = MultivariateTpeSampler::builder() +//! .gamma(0.15) +//! .n_startup_trials(20) +//! .group(true) +//! .seed(42) +//! .build() +//! .unwrap(); +//! let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler); +//! ``` mod gamma; mod multivariate; diff --git a/src/sampler/tpe/multivariate.rs b/src/sampler/tpe/multivariate.rs index 556545f..72ea840 100644 --- a/src/sampler/tpe/multivariate.rs +++ b/src/sampler/tpe/multivariate.rs @@ -167,15 +167,53 @@ pub enum ConstantLiarStrategy { /// A Multivariate Tree-Parzen Estimator (TPE) sampler for Bayesian optimization. /// -/// This sampler extends the standard TPE approach by modeling joint distributions -/// over all parameters, allowing it to capture parameter correlations. +/// Unlike the standard [`super::TpeSampler`], which samples each parameter +/// independently, this sampler models joint distributions over all parameters +/// using multivariate KDE. This captures correlations between parameters and +/// can significantly improve optimization on problems where parameters interact +/// (e.g., Rosenbrock, coupled hyperparameters). /// -/// # Fields +/// When the search space varies between trials (conditional parameters), the +/// sampler automatically falls back to independent TPE or uniform sampling for +/// parameters outside the intersection search space. /// -/// - `gamma_strategy`: Strategy for computing the gamma quantile -/// - `n_startup_trials`: Number of random trials before TPE sampling begins -/// - `n_ei_candidates`: Number of candidates to evaluate per joint sample -/// - `group`: Whether to decompose search space into independent groups +/// # When to use +/// +/// - Parameters are correlated or interact with each other. +/// - The search space is mostly fixed across trials. +/// - You need parallel optimization (enable [`ConstantLiarStrategy`]). +/// +/// Prefer [`super::TpeSampler`] when parameters are independent or the search space changes +/// dynamically. +/// +/// # Examples +/// +/// ``` +/// use optimizer::parameter::{FloatParam, Parameter}; +/// use optimizer::sampler::tpe::MultivariateTpeSampler; +/// use optimizer::{Direction, Study}; +/// +/// let sampler = MultivariateTpeSampler::builder() +/// .gamma(0.15) +/// .n_startup_trials(20) +/// .seed(42) +/// .build() +/// .unwrap(); +/// +/// let study: Study<f64> = Study::with_sampler(Direction::Minimize, sampler); +/// let x = FloatParam::new(-5.0, 5.0); +/// let y = FloatParam::new(-5.0, 5.0); +/// +/// study +/// .optimize(30, |trial| { +/// let xv = x.suggest(trial)?; +/// let yv = y.suggest(trial)?; +/// Ok::<_, optimizer::Error>(xv * xv + yv * yv) +/// }) +/// .unwrap(); +/// +/// assert!(study.best_value().unwrap() < 1.0); +/// ``` pub struct MultivariateTpeSampler { /// Strategy for computing the gamma quantile. gamma_strategy: Arc<dyn GammaStrategy>, diff --git a/src/sampler/tpe/sampler.rs b/src/sampler/tpe/sampler.rs index 6a40692..16ff43b 100644 --- a/src/sampler/tpe/sampler.rs +++ b/src/sampler/tpe/sampler.rs @@ -102,7 +102,7 @@ use crate::sampler::{CompletedTrial, Sampler}; /// /// // Create with custom settings using the builder /// let sampler = TpeSampler::builder() -/// .gamma(0.15) // Shorthand for Fixednew(0.15) +/// .gamma(0.15) // Shorthand for FixedGamma::new(0.15) /// .n_startup_trials(20) /// .n_ei_candidates(32) /// .seed(42) diff --git a/src/storage/journal.rs b/src/storage/journal.rs index 9e4ca68..367adfb 100644 --- a/src/storage/journal.rs +++ b/src/storage/journal.rs @@ -1,4 +1,72 @@ //! JSONL-based journal storage backend. +//! +//! [`JournalStorage`] persists completed trials as one JSON object per +//! line ([JSONL / JSON Lines](https://jsonlines.org/)) while keeping a +//! full copy in memory for fast read access. +//! +//! # File format +//! +//! Each line is a self-contained JSON serialization of a +//! [`CompletedTrial<V>`](crate::sampler::CompletedTrial). The file +//! is append-only — no existing lines are ever modified or deleted. +//! +//! ```text +//! {"id":0,"params":{...},"value":1.23,"state":"Completed",...} +//! {"id":1,"params":{...},"value":0.87,"state":"Completed",...} +//! ``` +//! +//! # File locking +//! +//! Concurrent access is coordinated with `fs2` file locks: +//! +//! - **Writes** acquire an *exclusive* lock so only one process +//! appends at a time. +//! - **Reads** ([`refresh`](super::Storage::refresh)) acquire a +//! *shared* lock so readers never see a partially written line. +//! +//! This makes it safe for multiple processes to share the same JSONL +//! file — for example, distributed workers each running their own +//! [`Study`](crate::Study) with a `JournalStorage` pointing to a +//! shared path. +//! +//! # Resuming a study +//! +//! Use [`JournalStorage::open`] to reload previously persisted trials +//! and continue optimization from where you left off: +//! +//! ```no_run +//! use optimizer::prelude::*; +//! use optimizer::storage::JournalStorage; +//! +//! // First run — creates the file. +//! let storage = JournalStorage::<f64>::new("trials.jsonl"); +//! let mut study = Study::builder().minimize().storage(storage).build(); +//! study +//! .optimize(50, |trial| { +//! let x = FloatParam::new(-5.0, 5.0).suggest(trial)?; +//! Ok::<_, optimizer::Error>(x * x) +//! }) +//! .unwrap(); +//! +//! // Later run — reloads previous 50 trials, then adds 50 more. +//! let storage = JournalStorage::<f64>::open("trials.jsonl").unwrap(); +//! let mut study = Study::builder().minimize().storage(storage).build(); +//! study +//! .optimize(50, |trial| { +//! let x = FloatParam::new(-5.0, 5.0).suggest(trial)?; +//! Ok::<_, optimizer::Error>(x * x) +//! }) +//! .unwrap(); +//! ``` +//! +//! # When to use +//! +//! - **Persistence** — survive process crashes or intentional restarts. +//! - **Multi-process** — several workers collaborating on a single study. +//! - **Inspection** — `cat trials.jsonl | jq .` for quick debugging. +//! +//! For pure in-memory usage without disk I/O, use +//! [`MemoryStorage`](super::MemoryStorage) instead (the default). use core::marker::PhantomData; use std::fs::{File, OpenOptions}; @@ -14,22 +82,30 @@ use serde::de::DeserializeOwned; use super::{MemoryStorage, Storage}; use crate::sampler::CompletedTrial; -/// A storage backend that appends completed trials as JSON lines to a file. +/// Append-only JSONL storage backend with file locking. /// -/// Trials are kept in memory for fast read access and simultaneously -/// persisted to a JSONL file. Multiple processes can safely share -/// the same file: writes use an exclusive file lock, reads use a -/// shared file lock. +/// Trials are kept in memory (via an inner [`MemoryStorage`]) for fast +/// read access and simultaneously appended to a JSONL file on disk. +/// Multiple processes can safely share the same file thanks to +/// `fs2` file locks — writes use an exclusive lock, reads use a +/// shared lock. /// -/// The type parameter `V` is the objective value type (typically `f64`). -/// It must be serializable so that trials can be written to disk. +/// The type parameter `V` is the objective value type (typically +/// `f64`). It must implement [`Serialize`](serde::Serialize) and +/// [`DeserializeOwned`](serde::de::DeserializeOwned) so trials can be +/// written to and read from disk. /// -/// # Examples +/// See the [`storage`](super) module docs for file format details +/// and a resumption example. +/// +/// # Example /// /// ```no_run +/// use optimizer::prelude::*; /// use optimizer::storage::JournalStorage; /// -/// let storage: JournalStorage<f64> = JournalStorage::new("trials.jsonl"); +/// let storage = JournalStorage::<f64>::new("trials.jsonl"); +/// let mut study = Study::builder().minimize().storage(storage).build(); /// ``` pub struct JournalStorage<V = f64> { memory: MemoryStorage<V>, @@ -40,14 +116,15 @@ pub struct JournalStorage<V = f64> { } impl<V: Serialize + DeserializeOwned + Send + Sync> JournalStorage<V> { - /// Creates a new journal storage that writes to the given path. + /// Create a new journal storage that writes to the given path. /// /// The file does not need to exist yet — it will be created on the /// first write. Existing trials in the file are **not** loaded /// until [`refresh`](Storage::refresh) is called (which happens /// automatically at the start of each trial via the [`Study`](crate::Study)). /// - /// To pre-load existing trials, use [`JournalStorage::open`]. + /// To pre-load existing trials at construction time, use + /// [`JournalStorage::open`] instead. #[must_use] pub fn new(path: impl AsRef<Path>) -> Self { Self { @@ -58,13 +135,14 @@ impl<V: Serialize + DeserializeOwned + Send + Sync> JournalStorage<V> { } } - /// Opens an existing journal file and loads all stored trials. + /// Open an existing journal file and load all stored trials. /// - /// If the file does not exist, returns an empty storage (no error). + /// If the file does not exist, return an empty storage (no error). + /// This is the primary way to **resume** a study after a restart. /// /// # Errors /// - /// Returns a [`Storage`](crate::Error::Storage) error if the file + /// Return a [`Storage`](crate::Error::Storage) error if the file /// exists but cannot be read or parsed. pub fn open(path: impl AsRef<Path>) -> crate::Result<Self> { let path = path.as_ref().to_path_buf(); diff --git a/src/storage/memory.rs b/src/storage/memory.rs index d9c2d29..676fc41 100644 --- a/src/storage/memory.rs +++ b/src/storage/memory.rs @@ -1,3 +1,32 @@ +//! In-memory storage backend. +//! +//! [`MemoryStorage`] is the default backend used by every +//! [`Study`](crate::Study). Trials are stored in a +//! `Vec<CompletedTrial<V>>` behind a [`parking_lot::RwLock`] for +//! thread-safe access. +//! +//! # When to use +//! +//! - **Single-process** studies where persistence is not needed. +//! - **Testing** or **prototyping** — zero configuration required. +//! - When you want the **fastest** possible read/write performance +//! (no disk I/O). +//! +//! For persistent storage that survives process restarts, see +//! [`JournalStorage`](super::JournalStorage) (requires the `journal` +//! feature). +//! +//! # Example +//! +//! ``` +//! use optimizer::prelude::*; +//! use optimizer::storage::MemoryStorage; +//! +//! // Explicit memory storage (equivalent to the default) +//! let storage = MemoryStorage::<f64>::new(); +//! let study = Study::builder().minimize().storage(storage).build(); +//! ``` + use core::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -8,14 +37,28 @@ use crate::sampler::CompletedTrial; /// In-memory trial storage (the default). /// -/// This is a thin wrapper around `Arc<RwLock<Vec<CompletedTrial<V>>>>`. +/// Wrap a `Vec<CompletedTrial<V>>` behind a read-write lock so that +/// trials can be appended from any thread. This is the backend that +/// [`Study`](crate::Study) uses when no explicit storage is provided. +/// +/// Use [`with_trials`](Self::with_trials) to seed a study with +/// previously collected data. +/// +/// # Example +/// +/// ``` +/// use optimizer::storage::{MemoryStorage, Storage}; +/// +/// let storage = MemoryStorage::<f64>::new(); +/// assert_eq!(storage.trials_arc().read().len(), 0); +/// ``` pub struct MemoryStorage<V> { trials: Arc<RwLock<Vec<CompletedTrial<V>>>>, next_id: AtomicU64, } impl<V> MemoryStorage<V> { - /// Creates a new, empty in-memory store. + /// Create a new, empty in-memory store. #[must_use] pub fn new() -> Self { Self { @@ -24,7 +67,10 @@ impl<V> MemoryStorage<V> { } } - /// Creates an in-memory store pre-populated with `trials`. + /// Create an in-memory store pre-populated with `trials`. + /// + /// The internal ID counter is set to one past the highest trial ID + /// so that subsequent trials receive unique IDs. #[must_use] pub fn with_trials(trials: Vec<CompletedTrial<V>>) -> Self { let next_id = trials.iter().map(|t| t.id).max().map_or(0, |id| id + 1); @@ -34,7 +80,8 @@ impl<V> MemoryStorage<V> { } } - /// Ensures the ID counter is at least `min_value`. + /// Ensure the ID counter is at least `min_value`. + #[cfg(feature = "journal")] pub(crate) fn bump_next_id(&self, min_value: u64) { self.next_id.fetch_max(min_value, Ordering::SeqCst); } diff --git a/src/storage/mod.rs b/src/storage/mod.rs index d5ceb00..1ed6357 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1,10 +1,42 @@ //! Trial storage backends. //! -//! The [`Storage`] trait defines how completed trials are stored and -//! accessed. [`MemoryStorage`] keeps trials in memory (the default). -//! With the `journal` feature enabled, [`JournalStorage`] appends -//! trials to a JSONL file with file-level locking so multiple -//! processes can safely share state. +//! The [`Storage`] trait defines how completed trials are persisted and +//! retrieved. Every [`Study`](crate::Study) owns an `Arc<dyn Storage<V>>` +//! so storage is transparently shared across threads. +//! +//! # Available backends +//! +//! | Backend | Description | Feature flag | +//! |---------|-------------|-------------| +//! | [`MemoryStorage`] | In-memory `Vec` behind a read-write lock (the default) | — | +//! | [`JournalStorage`] | JSONL file with `fs2` file locking for multi-process sharing | `journal` | +//! +//! # When to swap backends +//! +//! The default [`MemoryStorage`] is sufficient for single-process studies +//! where persistence is not needed. Switch to [`JournalStorage`] when you +//! want to: +//! +//! - **Resume** a study after a process restart. +//! - **Share state** across multiple processes writing to the same file. +//! - **Inspect** trial history in a human-readable JSONL file. +//! +//! # Implementing a custom backend +//! +//! Implement the [`Storage`] trait to plug in your own backend (e.g. a +//! database). The trait requires four methods: [`push`](Storage::push), +//! [`trials_arc`](Storage::trials_arc), [`next_trial_id`](Storage::next_trial_id), +//! and optionally [`refresh`](Storage::refresh) for external data sources. +//! +//! Inject your storage into a study via the builder: +//! +//! ``` +//! use optimizer::prelude::*; +//! use optimizer::storage::MemoryStorage; +//! +//! let storage = MemoryStorage::<f64>::new(); +//! let study = Study::builder().minimize().storage(storage).build(); +//! ``` #[cfg(feature = "journal")] mod journal; @@ -26,7 +58,8 @@ use crate::sampler::CompletedTrial; /// default implementation is [`MemoryStorage`], which keeps trials in /// a plain `Vec` behind a read-write lock. /// -/// Implementations must be safe to use from multiple threads. +/// Implementations must be `Send + Sync` because a study may be shared +/// across threads (e.g. via [`optimize_parallel`](crate::Study::optimize_parallel)). pub trait Storage<V>: Send + Sync { /// Append a completed trial to the store. fn push(&self, trial: CompletedTrial<V>); @@ -38,14 +71,14 @@ pub trait Storage<V>: Send + Sync { /// lock for efficient, allocation-free access. fn trials_arc(&self) -> &Arc<RwLock<Vec<CompletedTrial<V>>>>; - /// Atomically returns the next unique trial ID. + /// Atomically return the next unique trial ID. /// /// Each call increments an internal counter so that consecutive /// calls always produce distinct IDs. fn next_trial_id(&self) -> u64; /// Reload from an external source (e.g. a file written by another - /// process). Returns `true` if the in-memory buffer was updated. + /// process). Return `true` if the in-memory buffer was updated. /// /// The default implementation is a no-op that returns `false`. fn refresh(&self) -> bool { diff --git a/src/study.rs b/src/study.rs index 8577d5a..e8effcf 100644 --- a/src/study.rs +++ b/src/study.rs @@ -63,7 +63,7 @@ impl<V> Study<V> where V: PartialOrd, { - /// Creates a new study with the given optimization direction. + /// Create a new study with the given optimization direction. /// /// Uses the default `RandomSampler` for parameter sampling. /// @@ -87,7 +87,7 @@ where Self::with_sampler(direction, RandomSampler::new()) } - /// Returns a [`StudyBuilder`] for constructing a study with a fluent API. + /// Return a [`StudyBuilder`] for constructing a study with a fluent API. /// /// # Examples /// @@ -111,7 +111,7 @@ where } } - /// Creates a study that minimizes the objective value. + /// Create a study that minimizes the objective value. /// /// This is a shorthand for `Study::with_sampler(Direction::Minimize, sampler)`. /// @@ -136,7 +136,7 @@ where Self::with_sampler(Direction::Minimize, sampler) } - /// Creates a study that maximizes the objective value. + /// Create a study that maximizes the objective value. /// /// This is a shorthand for `Study::with_sampler(Direction::Maximize, sampler)`. /// @@ -161,7 +161,7 @@ where Self::with_sampler(Direction::Maximize, sampler) } - /// Creates a new study with a custom sampler. + /// Create a new study with a custom sampler. /// /// # Arguments /// @@ -189,7 +189,7 @@ where ) } - /// Builds a trial factory for sampler integration when `V = f64`. + /// Build a trial factory for sampler integration when `V = f64`. fn make_trial_factory( sampler: &Arc<dyn Sampler>, storage: &Arc<dyn crate::storage::Storage<V>>, @@ -220,10 +220,28 @@ where }) } - /// Creates a study with a custom sampler and storage backend. + /// Create a study with a custom sampler and storage backend. /// /// This is the most general constructor — all other constructors - /// delegate to this one. + /// delegate to this one. Use it when you need a non-default storage + /// backend (e.g., [`JournalStorage`](crate::storage::JournalStorage)). + /// + /// # Arguments + /// + /// * `direction` - Whether to minimize or maximize the objective function. + /// * `sampler` - The sampler to use for parameter sampling. + /// * `storage` - The storage backend for completed trials. + /// + /// # Examples + /// + /// ``` + /// use optimizer::sampler::random::RandomSampler; + /// use optimizer::storage::MemoryStorage; + /// use optimizer::{Direction, Study}; + /// + /// let storage = MemoryStorage::<f64>::new(); + /// let study = Study::with_sampler_and_storage(Direction::Minimize, RandomSampler::new(), storage); + /// ``` pub fn with_sampler_and_storage( direction: Direction, sampler: impl Sampler + 'static, @@ -247,28 +265,15 @@ where } } - /// Returns the optimization direction. + /// Return the optimization direction. #[must_use] pub fn direction(&self) -> Direction { self.direction } - /// Sets a new sampler for the study. + /// Creates a study with a custom sampler and pruner. /// - /// # Arguments - /// - /// * `sampler` - The sampler to use for parameter sampling. - /// - /// # Examples - /// - /// ``` - /// use optimizer::sampler::tpe::TpeSampler; - /// use optimizer::{Direction, Study}; - /// - /// let mut study: Study<f64> = Study::new(Direction::Minimize); - /// study.set_sampler(TpeSampler::new()); - /// ``` - /// Creates a new study with a custom sampler and pruner. + /// Uses the default [`MemoryStorage`](crate::storage::MemoryStorage) backend. /// /// # Arguments /// @@ -310,6 +315,21 @@ where } } + /// Replace the sampler used for future parameter suggestions. + /// + /// The new sampler takes effect for all subsequent calls to + /// [`create_trial`](Self::create_trial), [`ask`](Self::ask), and the + /// `optimize*` family. Already-completed trials are unaffected. + /// + /// # Examples + /// + /// ``` + /// use optimizer::sampler::tpe::TpeSampler; + /// use optimizer::{Direction, Study}; + /// + /// let mut study: Study<f64> = Study::new(Direction::Minimize); + /// study.set_sampler(TpeSampler::new()); + /// ``` pub fn set_sampler(&mut self, sampler: impl Sampler + 'static) where V: 'static, @@ -318,11 +338,18 @@ where self.trial_factory = Self::make_trial_factory(&self.sampler, &self.storage, &self.pruner); } - /// Sets a new pruner for the study. + /// Replace the pruner used for future trials. /// - /// # Arguments + /// The new pruner takes effect for all trials created after this call. /// - /// * `pruner` - The pruner to use for trial pruning. + /// # Examples + /// + /// ``` + /// use optimizer::prelude::*; + /// + /// let mut study: Study<f64> = Study::new(Direction::Minimize); + /// study.set_pruner(MedianPruner::new(Direction::Minimize)); + /// ``` pub fn set_pruner(&mut self, pruner: impl Pruner + 'static) where V: 'static, @@ -331,13 +358,13 @@ where self.trial_factory = Self::make_trial_factory(&self.sampler, &self.storage, &self.pruner); } - /// Returns a reference to the study's pruner. + /// Return a reference to the study's current pruner. #[must_use] pub fn pruner(&self) -> &dyn Pruner { &*self.pruner } - /// Enqueues a specific parameter configuration to be evaluated next. + /// Enqueue a specific parameter configuration to be evaluated next. /// /// The next call to [`ask()`](Self::ask) or the next trial in [`optimize()`](Self::optimize) /// will use these exact parameters instead of sampling from the sampler. @@ -377,7 +404,7 @@ where self.enqueued_params.lock().push_back(params); } - /// Returns the trial ID of the current best trial from the given slice. + /// Return the trial ID of the current best trial from the given slice. #[cfg(feature = "tracing")] fn best_id(&self, trials: &[CompletedTrial<V>]) -> Option<u64> { let direction = self.direction; @@ -388,7 +415,7 @@ where .map(|t| t.id) } - /// Creates a new trial with pre-set parameter values. + /// Create a new trial with pre-set parameter values. /// /// The trial gets a new unique ID but reuses the given parameters. When /// `suggest_param` is called on the resulting trial, fixed values are @@ -404,18 +431,20 @@ where trial } - /// Returns the number of enqueued parameter configurations. + /// Return the number of enqueued parameter configurations. + /// + /// See [`enqueue`](Self::enqueue) for how to add configurations. #[must_use] pub fn n_enqueued(&self) -> usize { self.enqueued_params.lock().len() } - /// Generates the next unique trial ID. + /// Generate the next unique trial ID. pub(crate) fn next_trial_id(&self) -> u64 { self.storage.next_trial_id() } - /// Creates a new trial with a unique ID. + /// Create a new trial with a unique ID. /// /// The trial starts in the `Running` state and can be used to suggest /// parameter values. After the objective function is evaluated, call @@ -456,7 +485,7 @@ where trial } - /// Records a completed trial with its objective value. + /// Record a completed trial with its objective value. /// /// This method stores the trial's parameters, distributions, and objective /// value in the study's history. The stored data is used by samplers to @@ -499,7 +528,7 @@ where self.storage.push(completed); } - /// Records a failed trial with an error message. + /// Record a failed trial with an error message. /// /// Failed trials are not stored in the study's history and do not /// contribute to future sampling decisions. This method is useful @@ -582,11 +611,15 @@ where } } - /// Records a pruned trial, preserving its intermediate values. + /// Record a pruned trial, preserving its intermediate values. /// /// Pruned trials are stored alongside completed trials so that samplers /// can optionally learn from partial evaluations. The trial's state is - /// set to `Pruned`. + /// set to [`Pruned`](crate::TrialState::Pruned). + /// + /// In practice you rarely call this directly — returning + /// `Err(TrialPruned)` from an objective function handles pruning + /// automatically. /// /// # Arguments /// @@ -611,9 +644,9 @@ where self.storage.push(completed); } - /// Returns an iterator over all completed trials. + /// Return all completed trials as a `Vec`. /// - /// The iterator yields references to `CompletedTrial` values, which contain + /// The returned vector contains clones of `CompletedTrial` values, which contain /// the trial's parameters, distributions, and objective value. /// /// Note: This method acquires a read lock on the completed trials, so the @@ -643,7 +676,7 @@ where self.storage.trials_arc().read().clone() } - /// Returns the number of completed trials. + /// Return the number of completed trials. /// /// Failed trials are not counted. /// @@ -667,7 +700,9 @@ where self.storage.trials_arc().read().len() } - /// Returns the number of pruned trials. + /// Return the number of pruned trials. + /// + /// Pruned trials are those that were stopped early by the pruner. #[must_use] pub fn n_pruned_trials(&self) -> usize { self.storage @@ -678,7 +713,7 @@ where .count() } - /// Compares two completed trials using constraint-aware ranking. + /// Compare two completed trials using constraint-aware ranking. /// /// 1. Feasible trials always rank above infeasible trials. /// 2. Among feasible trials, rank by objective value (respecting direction). @@ -708,7 +743,7 @@ where } } - /// Returns the trial with the best objective value. + /// Return the trial with the best objective value. /// /// The "best" trial depends on the optimization direction: /// - `Direction::Minimize`: Returns the trial with the lowest objective value. @@ -762,7 +797,7 @@ where Ok(best.clone()) } - /// Returns the best objective value found so far. + /// Return the best objective value found so far. /// /// The "best" value depends on the optimization direction: /// - `Direction::Minimize`: Returns the lowest objective value. @@ -803,13 +838,33 @@ where self.best_trial().map(|trial| trial.value) } - /// Returns the top `n` trials sorted by objective value. + /// Return the top `n` trials sorted by objective value. /// /// For `Direction::Minimize`, returns trials with the lowest values. /// For `Direction::Maximize`, returns trials with the highest values. /// Only includes completed trials (not failed or pruned). /// /// If fewer than `n` completed trials exist, returns all of them. + /// + /// # Examples + /// + /// ``` + /// use optimizer::parameter::{FloatParam, Parameter}; + /// use optimizer::{Direction, Study}; + /// + /// let study: Study<f64> = Study::new(Direction::Minimize); + /// let x = FloatParam::new(0.0, 10.0); + /// + /// for val in [5.0, 1.0, 3.0] { + /// let mut t = study.create_trial(); + /// let _ = x.suggest(&mut t); + /// study.complete_trial(t, val); + /// } + /// + /// let top2 = study.top_trials(2); + /// assert_eq!(top2.len(), 2); + /// assert!(top2[0].value <= top2[1].value); + /// ``` #[must_use] pub fn top_trials(&self, n: usize) -> Vec<CompletedTrial<V>> where @@ -828,7 +883,7 @@ where completed } - /// Runs optimization with the given objective function. + /// Run optimization with the given objective function. /// /// This method runs `n_trials` evaluations sequentially. For each trial: /// 1. A new trial is created @@ -936,7 +991,7 @@ where Ok(()) } - /// Runs optimization asynchronously with the given objective function. + /// Run optimization asynchronously with the given objective function. /// /// This method runs `n_trials` evaluations sequentially, but the objective /// function can be async (e.g., for I/O-bound operations like network requests @@ -1036,7 +1091,7 @@ where Ok(()) } - /// Runs optimization with bounded parallelism for concurrent trial evaluation. + /// Run optimization with bounded parallelism for concurrent trial evaluation. /// /// This method runs up to `concurrency` trials simultaneously, allowing /// efficient use of async I/O-bound objective functions. A semaphore limits @@ -1163,7 +1218,7 @@ where Ok(()) } - /// Runs optimization with a callback for monitoring progress. + /// Run optimization with a callback for monitoring progress. /// /// This method is similar to `optimize`, but calls a callback function after /// each completed trial. The callback can inspect the study state and the @@ -1305,7 +1360,7 @@ where Ok(()) } - /// Runs optimization until the given duration has elapsed. + /// Run optimization until the given duration has elapsed. /// /// Trials that are already running when the timeout is reached will /// complete — we never interrupt mid-trial. The actual elapsed time @@ -1392,7 +1447,7 @@ where Ok(()) } - /// Runs optimization until the given duration has elapsed, with a callback. + /// Run optimization until the given duration has elapsed, with a callback. /// /// Like [`optimize_until`](Self::optimize_until), but calls a callback after /// each completed trial. The callback can stop optimization early by returning @@ -1526,16 +1581,17 @@ where Ok(()) } - /// Runs optimization asynchronously until the given duration has elapsed. + /// Run optimization asynchronously until the given duration has elapsed. /// /// The async variant of [`optimize_until`](Self::optimize_until). Trials are - /// run sequentially, but the objective function can be async. + /// run sequentially, but the objective function can be async (useful for + /// I/O-bound evaluations). /// /// # Arguments /// /// * `duration` - The maximum wall-clock time to spend on optimization. /// * `objective` - A function that takes a `Trial` and returns a `Future` - /// that resolves to a tuple of `(Trial, Result<V, E>)`. + /// that resolves to a tuple of `(Trial, V)` or an error. /// /// # Errors /// @@ -1585,7 +1641,7 @@ where Ok(()) } - /// Runs optimization with bounded parallelism until the given duration has elapsed. + /// Run optimization with bounded parallelism until the given duration has elapsed. /// /// The parallel variant of [`optimize_until`](Self::optimize_until). Runs up to /// `concurrency` trials simultaneously using async tasks. New trials are spawned @@ -1675,7 +1731,7 @@ where Ok(()) } - /// Runs optimization with automatic retry for failed trials. + /// Run optimization with automatic retry for failed trials. /// /// If the objective function returns an error, the same parameter /// configuration is retried up to `max_retries` times. Only after all @@ -1789,7 +1845,7 @@ impl<V> Study<V> where V: PartialOrd + Clone + fmt::Display, { - /// Export completed trials to CSV format. + /// Write completed trials to a writer in CSV format. /// /// Columns: `trial_id`, `value`, `state`, then one column per unique /// parameter label, then one column per unique user-attribute key. @@ -1800,6 +1856,25 @@ where /// # Errors /// /// Returns an I/O error if writing fails. + /// + /// # Examples + /// + /// ``` + /// use optimizer::parameter::{FloatParam, Parameter}; + /// use optimizer::{Direction, Study}; + /// + /// let study: Study<f64> = Study::new(Direction::Minimize); + /// let x = FloatParam::new(0.0, 10.0).name("x"); + /// + /// let mut trial = study.create_trial(); + /// let _ = x.suggest(&mut trial); + /// study.complete_trial(trial, 0.42); + /// + /// let mut buf = Vec::new(); + /// study.to_csv(&mut buf).unwrap(); + /// let csv = String::from_utf8(buf).unwrap(); + /// assert!(csv.contains("trial_id")); + /// ``` pub fn to_csv(&self, mut writer: impl std::io::Write) -> std::io::Result<()> { use std::collections::BTreeMap; @@ -1892,7 +1967,10 @@ where Ok(()) } - /// Export completed trials to a CSV file. + /// Export completed trials to a CSV file at the given path. + /// + /// Convenience wrapper around [`to_csv`](Self::to_csv) that creates a + /// buffered file writer. /// /// # Errors /// @@ -1902,7 +1980,7 @@ where self.to_csv(std::io::BufWriter::new(file)) } - /// Returns a human-readable summary of the study. + /// Return a human-readable summary of the study. /// /// The summary includes: /// - Optimization direction and total trial count @@ -1973,10 +2051,24 @@ impl<V> Study<V> where V: PartialOrd + Clone, { - /// Returns an iterator over all completed trials. + /// Return an iterator over all completed trials. /// /// This clones the internal trial list, so it is suitable for /// analysis and iteration but not for hot paths. + /// + /// # Examples + /// + /// ``` + /// use optimizer::{Direction, Study}; + /// + /// let study: Study<f64> = Study::new(Direction::Minimize); + /// let trial = study.create_trial(); + /// study.complete_trial(trial, 1.0); + /// + /// for t in study.iter() { + /// println!("Trial {} → {}", t.id, t.value); + /// } + /// ``` #[must_use] pub fn iter(&self) -> std::vec::IntoIter<CompletedTrial<V>> { self.trials().into_iter() @@ -1987,7 +2079,7 @@ impl<V> Study<V> where V: PartialOrd + Clone + Into<f64>, { - /// Computes parameter importance scores using Spearman rank correlation. + /// Compute parameter importance scores using Spearman rank correlation. /// /// For each parameter, the absolute Spearman correlation between its values /// and the objective values is computed across all completed trials. Scores @@ -2086,7 +2178,7 @@ where scores } - /// Computes parameter importance using fANOVA (functional ANOVA) with + /// Compute parameter importance using fANOVA (functional ANOVA) with /// default configuration. /// /// Fits a random forest to the trial data and decomposes variance into @@ -2097,11 +2189,33 @@ where /// # Errors /// /// Returns [`crate::Error::NoCompletedTrials`] if fewer than 2 trials have completed. + /// + /// # Examples + /// + /// ``` + /// use optimizer::parameter::{FloatParam, Parameter}; + /// use optimizer::{Direction, Study}; + /// + /// let study: Study<f64> = Study::new(Direction::Minimize); + /// let x = FloatParam::new(0.0, 10.0).name("x"); + /// let y = FloatParam::new(0.0, 10.0).name("y"); + /// + /// study + /// .optimize(30, |trial| { + /// let xv = x.suggest(trial)?; + /// let yv = y.suggest(trial)?; + /// Ok::<_, optimizer::Error>(xv * xv + 0.1 * yv) + /// }) + /// .unwrap(); + /// + /// let result = study.fanova().unwrap(); + /// assert!(!result.main_effects.is_empty()); + /// ``` pub fn fanova(&self) -> crate::Result<crate::fanova::FanovaResult> { self.fanova_with_config(&crate::fanova::FanovaConfig::default()) } - /// Computes parameter importance using fANOVA with custom configuration. + /// Compute parameter importance using fANOVA with custom configuration. /// /// See [`Self::fanova`] for details. The [`FanovaConfig`](crate::fanova::FanovaConfig) /// allows tuning the number of trees, tree depth, and random seed. @@ -2316,7 +2430,30 @@ impl Study<f64> { } impl<V: PartialOrd + Send + Sync + 'static> Study<V> { - /// Creates a study with a custom sampler, pruner, and storage backend. + /// Create a study with a custom sampler, pruner, and storage backend. + /// + /// The most flexible constructor, allowing full control over all components. + /// + /// # Arguments + /// + /// * `direction` - Whether to minimize or maximize the objective function. + /// * `sampler` - The sampler to use for parameter sampling. + /// * `pruner` - The pruner to use for trial pruning. + /// * `storage` - The storage backend for completed trials. + /// + /// # Examples + /// + /// ``` + /// use optimizer::prelude::*; + /// use optimizer::storage::MemoryStorage; + /// + /// let study = Study::with_sampler_pruner_and_storage( + /// Direction::Minimize, + /// TpeSampler::new(), + /// MedianPruner::new(Direction::Minimize), + /// MemoryStorage::<f64>::new(), + /// ); + /// ``` pub fn with_sampler_pruner_and_storage( direction: Direction, sampler: impl Sampler + 'static, @@ -2373,49 +2510,55 @@ pub struct StudyBuilder<V: PartialOrd = f64> { } impl<V: PartialOrd> StudyBuilder<V> { - /// Sets the optimization direction to minimize. + /// Set the optimization direction to minimize (the default). #[must_use] pub fn minimize(mut self) -> Self { self.direction = Direction::Minimize; self } - /// Sets the optimization direction to maximize. + /// Set the optimization direction to maximize. #[must_use] pub fn maximize(mut self) -> Self { self.direction = Direction::Maximize; self } - /// Sets the optimization direction. + /// Set the optimization direction explicitly. #[must_use] pub fn direction(mut self, direction: Direction) -> Self { self.direction = direction; self } - /// Sets the sampler used for parameter suggestions. + /// Set the sampler used for parameter suggestions. + /// + /// Defaults to [`RandomSampler`] if not specified. #[must_use] pub fn sampler(mut self, sampler: impl Sampler + 'static) -> Self { self.sampler = Some(Box::new(sampler)); self } - /// Sets the pruner used for early stopping of trials. + /// Set the pruner used for early stopping of trials. + /// + /// Defaults to [`NopPruner`] (no pruning) if not specified. #[must_use] pub fn pruner(mut self, pruner: impl Pruner + 'static) -> Self { self.pruner = Some(Box::new(pruner)); self } - /// Sets a custom storage backend. + /// Set a custom storage backend. + /// + /// Defaults to [`MemoryStorage`](crate::storage::MemoryStorage) if not specified. #[must_use] pub fn storage(mut self, storage: impl crate::storage::Storage<V> + 'static) -> Self { self.storage = Some(Box::new(storage)); self } - /// Builds the [`Study`] with the configured options. + /// Build the [`Study`] with the configured options. #[must_use] pub fn build(self) -> Study<V> where @@ -2450,15 +2593,31 @@ impl<V> Study<V> where V: PartialOrd + Send + Sync + serde::Serialize + serde::de::DeserializeOwned + 'static, { - /// Creates a study backed by a JSONL journal file. + /// Create a study backed by a JSONL journal file. /// /// Any existing trials in the file are loaded into memory and the /// trial ID counter is set to one past the highest stored ID. New /// trials are written through to the file on completion. /// + /// # Arguments + /// + /// * `direction` - Whether to minimize or maximize the objective function. + /// * `sampler` - The sampler to use for parameter sampling. + /// * `path` - Path to the JSONL journal file (created if absent). + /// /// # Errors /// /// Returns a [`Storage`](crate::Error::Storage) error if loading fails. + /// + /// # Examples + /// + /// ```no_run + /// use optimizer::sampler::tpe::TpeSampler; + /// use optimizer::{Direction, Study}; + /// + /// let study: Study<f64> = + /// Study::with_journal(Direction::Minimize, TpeSampler::new(), "trials.jsonl").unwrap(); + /// ``` pub fn with_journal( direction: Direction, sampler: impl Sampler + 'static, @@ -2470,9 +2629,9 @@ where } impl Study<f64> { - /// Generates an HTML report with interactive Plotly.js charts. + /// Generate an HTML report with interactive Plotly.js charts. /// - /// Creates a self-contained HTML file that can be opened in any browser. + /// Create a self-contained HTML file that can be opened in any browser. /// See [`generate_html_report`](crate::visualization::generate_html_report) /// for details on the included charts. /// @@ -2519,6 +2678,7 @@ impl<V: PartialOrd + Clone + serde::Serialize> Study<V> { /// Export trials as a pretty-printed JSON array to a file. /// /// Each element in the array is a serialized [`CompletedTrial`]. + /// Requires the `serde` feature. /// /// # Errors /// @@ -2529,7 +2689,7 @@ impl<V: PartialOrd + Clone + serde::Serialize> Study<V> { serde_json::to_writer_pretty(file, &trials).map_err(std::io::Error::other) } - /// Saves the study state to a JSON file. + /// Save the study state to a JSON file. /// /// # Errors /// @@ -2561,7 +2721,7 @@ impl<V: PartialOrd + Clone + serde::Serialize> Study<V> { #[cfg(feature = "serde")] impl<V: PartialOrd + Clone + Default + serde::Serialize> Study<V> { - /// Runs optimization with automatic checkpointing every `interval` trials. + /// Run optimization with automatic checkpointing every `interval` trials. /// /// This is convenience sugar over [`optimize_with_callback`](Self::optimize_with_callback) /// combined with [`save`](Self::save). The checkpoint is written atomically so @@ -2595,7 +2755,7 @@ impl<V: PartialOrd + Clone + Default + serde::Serialize> Study<V> { #[cfg(feature = "serde")] impl<V: PartialOrd + Send + Sync + Clone + serde::de::DeserializeOwned + 'static> Study<V> { - /// Loads a study from a JSON file. + /// Load a study from a JSON file. /// /// The loaded study uses a `RandomSampler` by default. Call /// [`set_sampler()`](Self::set_sampler) to restore the original sampler diff --git a/src/trial.rs b/src/trial.rs index 4f90fd3..e46077e 100644 --- a/src/trial.rs +++ b/src/trial.rs @@ -1,4 +1,23 @@ -//! Trial implementation for tracking sampled parameters and trial state. +//! Trial lifecycle management for optimization runs. +//! +//! A [`Trial`] represents a single evaluation of the objective function. The study +//! creates trials, the objective function samples parameters from them via +//! [`Parameter::suggest`](crate::parameter::Parameter::suggest), and reports +//! intermediate values for pruning decisions. +//! +//! # Lifecycle +//! +//! 1. **Created** — `Study` creates a trial with [`Trial::new`] or internally via +//! `Trial::with_sampler`. +//! 2. **Running** — The objective calls [`Trial::suggest_param`] to sample parameters +//! and optionally [`Trial::report`] / [`Trial::should_prune`] for early stopping. +//! 3. **Completed / Failed / Pruned** — The study marks the trial's final state. +//! +//! # User Attributes +//! +//! Trials support arbitrary key-value metadata via [`Trial::set_user_attr`] and +//! [`Trial::user_attr`], useful for logging hyperparameters, hardware info, or +//! debug notes alongside the optimization results. use std::collections::HashMap; use std::sync::Arc; @@ -57,14 +76,26 @@ impl From<bool> for AttrValue { } } -/// A trial represents a single evaluation of the objective function. +/// A single evaluation of the objective function. /// /// Each trial has a unique ID and stores the sampled parameters along with -/// their distributions. The trial progresses through states: Running -> Complete/Failed. +/// their distributions. The trial progresses through states: +/// `Running` → `Complete` / `Failed` / `Pruned`. /// -/// Trials use a sampler to generate parameter values. When created through -/// `Study::create_trial()`, the trial receives the study's sampler and access -/// to the history of completed trials for informed sampling. +/// Trials use a [`Sampler`](crate::sampler::Sampler) to generate parameter +/// values. When created through [`Study::create_trial`](crate::Study::create_trial), +/// the trial receives the study's sampler and access to the history of +/// completed trials for informed sampling. +/// +/// # Examples +/// +/// ``` +/// use optimizer::Trial; +/// use optimizer::parameter::{FloatParam, Parameter}; +/// +/// let mut trial = Trial::new(0); +/// let x = FloatParam::new(-5.0, 5.0).suggest(&mut trial).unwrap(); +/// ``` #[derive(Clone)] pub struct Trial { /// Unique identifier for this trial. @@ -113,13 +144,14 @@ impl core::fmt::Debug for Trial { } impl Trial { - /// Creates a new trial with the given ID. + /// Create a new trial with the given ID. /// /// The trial starts in the `Running` state with no parameters sampled. - /// This constructor creates a trial without a sampler, which will use - /// local random sampling for suggest methods. + /// This constructor creates a trial without a sampler, which will fall + /// back to random sampling for [`suggest_param`](Self::suggest_param) calls. /// - /// For trials that use the study's sampler, use `Trial::with_sampler` instead. + /// For trials that use the study's sampler, the study creates them + /// internally via `Trial::with_sampler`. /// /// # Arguments /// @@ -151,10 +183,10 @@ impl Trial { } } - /// Creates a new trial with a sampler and access to trial history. + /// Create a new trial with a sampler and access to trial history. /// - /// This constructor is used by `Study::create_trial()` to create trials - /// that use the study's sampler for informed parameter suggestions. + /// Used internally by `Study::create_trial()` to create trials that use + /// the study's sampler for informed parameter suggestions. /// /// # Arguments /// @@ -183,19 +215,19 @@ impl Trial { } } - /// Sets pre-filled parameters on this trial. + /// Set pre-filled parameters on this trial. /// - /// When `suggest_param` is called for a parameter that has a fixed value, - /// the fixed value is used instead of sampling. + /// When [`suggest_param`](Self::suggest_param) is called for a parameter + /// that has a fixed value, the fixed value is used instead of sampling. pub(crate) fn set_fixed_params(&mut self, params: HashMap<ParamId, ParamValue>) { self.fixed_params = params; } - /// Samples a value from the given distribution using the sampler. + /// Sample a value from the given distribution using the sampler. /// - /// If the trial has a sampler, it delegates to the sampler's sample method - /// with the history of completed trials. Otherwise, it uses the `RandomSampler` - /// as a fallback. + /// If the trial has a sampler, delegates to the sampler's sample method + /// with the history of completed trials. Otherwise, falls back to + /// [`RandomSampler`](crate::sampler::random::RandomSampler). fn sample_value(&self, distribution: &Distribution) -> ParamValue { if let (Some(sampler), Some(history)) = (&self.sampler, &self.history) { let history_guard = history.read(); @@ -208,40 +240,55 @@ impl Trial { } } - /// Returns the unique ID of this trial. + /// Return the unique ID of this trial. #[must_use] pub fn id(&self) -> u64 { self.id } - /// Returns the current state of this trial. + /// Return the current state of this trial. #[must_use] pub fn state(&self) -> TrialState { self.state } - /// Returns a reference to the sampled parameters. + /// Return a reference to the sampled parameters, keyed by [`ParamId`](crate::parameter::ParamId). #[must_use] pub fn params(&self) -> &HashMap<ParamId, ParamValue> { &self.params } - /// Returns a reference to the parameter distributions. + /// Return a reference to the parameter distributions, keyed by [`ParamId`](crate::parameter::ParamId). #[must_use] pub fn distributions(&self) -> &HashMap<ParamId, Distribution> { &self.distributions } - /// Returns a reference to the parameter labels. + /// Return a reference to the parameter labels, keyed by [`ParamId`](crate::parameter::ParamId). #[must_use] pub fn param_labels(&self) -> &HashMap<ParamId, String> { &self.param_labels } - /// Reports an intermediate objective value at a given step. + /// Report an intermediate objective value at a given step. /// - /// Steps should be monotonically increasing (e.g., epoch number). - /// Duplicate steps overwrite the previous value. + /// Call this during iterative training (e.g., once per epoch) so the + /// [`Pruner`](crate::pruner::Pruner) can decide whether to stop the trial + /// early. Steps should be monotonically increasing; duplicate steps + /// overwrite the previous value. + /// + /// # Examples + /// + /// ``` + /// use optimizer::Trial; + /// + /// let mut trial = Trial::new(0); + /// for epoch in 0..10 { + /// let loss = 1.0 / (epoch as f64 + 1.0); + /// trial.report(epoch, loss); + /// } + /// assert_eq!(trial.intermediate_values().len(), 10); + /// ``` pub fn report(&mut self, step: u64, value: f64) { if let Some(entry) = self .intermediate_values @@ -256,8 +303,11 @@ impl Trial { /// Ask whether this trial should be pruned at the current step. /// - /// Returns `true` if the pruner recommends stopping this trial. - /// The caller should return `Err(TrialPruned)` from the objective. + /// Return `true` if the pruner recommends stopping this trial based on + /// the intermediate values reported so far. When `true`, the objective + /// should return early with `Err(TrialPruned)?`. + /// + /// Always returns `false` when no pruner is configured. #[must_use] pub fn should_prune(&self) -> bool { let (Some(pruner), Some(history)) = (&self.pruner, &self.history) else { @@ -274,59 +324,87 @@ impl Trial { prune } - /// Returns all intermediate values reported so far. + /// Return all intermediate values reported so far as `(step, value)` pairs. #[must_use] pub fn intermediate_values(&self) -> &[(u64, f64)] { &self.intermediate_values } - /// Sets a user attribute on this trial. + /// Set a user attribute on this trial. + /// + /// User attributes are arbitrary key-value pairs for logging, debugging, + /// or analysis. Values can be `f64`, `i64`, `String`, `&str`, or `bool` + /// (anything implementing `Into<AttrValue>`). + /// + /// # Examples + /// + /// ``` + /// use optimizer::Trial; + /// + /// let mut trial = Trial::new(0); + /// trial.set_user_attr("gpu", "A100"); + /// trial.set_user_attr("batch_size", 64_i64); + /// trial.set_user_attr("accuracy", 0.95); + /// ``` pub fn set_user_attr(&mut self, key: impl Into<String>, value: impl Into<AttrValue>) { self.user_attrs.insert(key.into(), value.into()); } - /// Gets a user attribute by key. + /// Return a user attribute by key, or `None` if it does not exist. #[must_use] pub fn user_attr(&self, key: &str) -> Option<&AttrValue> { self.user_attrs.get(key) } - /// Returns all user attributes. + /// Return all user attributes as a map. #[must_use] pub fn user_attrs(&self) -> &HashMap<String, AttrValue> { &self.user_attrs } - /// Sets constraint values for this trial. + /// Set constraint values for this trial. /// - /// Each value represents a constraint; a value <= 0.0 means the constraint - /// is satisfied (feasible). A value > 0.0 means the constraint is violated. + /// Each element represents one constraint. A value ≤ 0.0 means the + /// constraint is satisfied (feasible); a value > 0.0 means violated. + /// Constrained samplers (e.g., NSGA-II with constraints) use these values + /// to prefer feasible solutions. + /// + /// # Examples + /// + /// ``` + /// use optimizer::Trial; + /// + /// let mut trial = Trial::new(0); + /// // Two constraints: first satisfied, second violated + /// trial.set_constraints(vec![-0.5, 0.3]); + /// assert_eq!(trial.constraint_values(), &[-0.5, 0.3]); + /// ``` pub fn set_constraints(&mut self, values: Vec<f64>) { self.constraint_values = values; } - /// Returns the constraint values for this trial. + /// Return the constraint values for this trial. #[must_use] pub fn constraint_values(&self) -> &[f64] { &self.constraint_values } - /// Sets the trial state to Complete. + /// Set the trial state to `Complete`. pub(crate) fn set_complete(&mut self) { self.state = TrialState::Complete; } - /// Sets the trial state to Failed. + /// Set the trial state to `Failed`. pub(crate) fn set_failed(&mut self) { self.state = TrialState::Failed; } - /// Sets the trial state to Pruned. + /// Set the trial state to `Pruned`. pub(crate) fn set_pruned(&mut self) { self.state = TrialState::Pruned; } - /// Suggests a parameter value using a [`Parameter`] definition. + /// Suggest a parameter value using a [`Parameter`] definition. /// /// This is the primary entry point for sampling parameters. It handles /// validation, caching, conflict detection, sampling, and conversion. diff --git a/src/visualization.rs b/src/visualization.rs index 3f1a69e..f607f25 100644 --- a/src/visualization.rs +++ b/src/visualization.rs @@ -1,7 +1,41 @@ //! HTML report generation for optimization visualization. //! -//! Generates self-contained HTML files with embedded Plotly.js charts -//! for offline visualization of optimization results. +//! Generate self-contained HTML files with embedded +//! [Plotly.js](https://plotly.com/javascript/) charts for offline +//! visualization of optimization results. No feature flag is required — +//! this module is always available. +//! +//! # Charts included +//! +//! | Chart | Description | +//! |---|---| +//! | **Optimization history** | Objective value vs trial number with best-so-far line | +//! | **Slice plots** | Objective value vs each parameter (1D scatter per param) | +//! | **Parallel coordinates** | Multi-parameter relationship view (color = objective) | +//! | **Parameter importance** | Horizontal bar chart of Spearman-based importance | +//! | **Trial timeline** | Duration/index of each trial, color-coded by state | +//! | **Intermediate values** | Per-trial learning curves (if pruning data available) | +//! +//! # Usage +//! +//! Call [`Study::export_html()`](crate::Study::export_html) or +//! [`generate_html_report()`] directly: +//! +//! ```no_run +//! use optimizer::prelude::*; +//! +//! let study: Study<f64> = Study::new(Direction::Minimize); +//! # let x = FloatParam::new(0.0, 1.0); +//! # study.optimize(10, |trial| { +//! # let v = x.suggest(trial)?; +//! # Ok::<_, optimizer::Error>(v * v) +//! # }).unwrap(); +//! study.export_html("report.html").unwrap(); +//! ``` +//! +//! The output is a single HTML file that can be opened in any browser. +//! An internet connection is needed on first load to fetch `Plotly.js` +//! from a CDN. use core::fmt::Write as _; use std::collections::BTreeMap; @@ -15,17 +49,19 @@ use crate::types::{Direction, TrialState}; /// Generate an HTML report with interactive Plotly.js charts. /// -/// Creates a self-contained HTML file at `path` containing: -/// - **Optimization history**: Objective value vs trial number with best-so-far line -/// - **Slice plots**: Objective value vs each parameter (1D scatter) -/// - **Parallel coordinates**: Multi-parameter relationship view -/// - **Trial timeline**: Duration index of each trial (horizontal bar) -/// - **Intermediate values**: Learning curves per trial (if pruning data available) -/// - **Parameter importance**: Bar chart (if enough completed trials) +/// Create a self-contained HTML file at `path` containing up to six +/// interactive charts. Charts that require data not present in the study +/// (e.g., intermediate values) are automatically omitted. +/// +/// The report includes: optimization history, slice plots, parallel +/// coordinates, parameter importance, trial timeline, and intermediate +/// values (when available). +/// +/// This is also available as [`Study::export_html()`](crate::Study::export_html). /// /// # Errors /// -/// Returns an I/O error if the file cannot be created or written. +/// Return an I/O error if the file cannot be created or written. pub fn generate_html_report( study: &crate::Study<f64>, path: impl AsRef<Path>,