feat: add trial user attributes for logging and analysis
Add AttrValue enum (Float, Int, String, Bool) with From impls and set_user_attr/user_attr/user_attrs methods on both Trial and CompletedTrial. Attributes set during optimization are propagated through complete_trial and prune_trial. AttrValue is re-exported at crate root and in the prelude.
This commit is contained in:
+2
-2
@@ -210,7 +210,7 @@ pub use sampler::grid::GridSearchSampler;
|
|||||||
pub use sampler::random::RandomSampler;
|
pub use sampler::random::RandomSampler;
|
||||||
pub use sampler::tpe::TpeSampler;
|
pub use sampler::tpe::TpeSampler;
|
||||||
pub use study::Study;
|
pub use study::Study;
|
||||||
pub use trial::Trial;
|
pub use trial::{AttrValue, Trial};
|
||||||
pub use types::{Direction, TrialState};
|
pub use types::{Direction, TrialState};
|
||||||
|
|
||||||
/// Convenient wildcard import for the most common types.
|
/// Convenient wildcard import for the most common types.
|
||||||
@@ -236,6 +236,6 @@ pub mod prelude {
|
|||||||
pub use crate::sampler::random::RandomSampler;
|
pub use crate::sampler::random::RandomSampler;
|
||||||
pub use crate::sampler::tpe::TpeSampler;
|
pub use crate::sampler::tpe::TpeSampler;
|
||||||
pub use crate::study::Study;
|
pub use crate::study::Study;
|
||||||
pub use crate::trial::Trial;
|
pub use crate::trial::{AttrValue, Trial};
|
||||||
pub use crate::types::Direction;
|
pub use crate::types::Direction;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -278,6 +278,7 @@ mod tests {
|
|||||||
HashMap::new(),
|
HashMap::new(),
|
||||||
0.0,
|
0.0,
|
||||||
values.to_vec(),
|
values.to_vec(),
|
||||||
|
HashMap::new(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -204,6 +204,7 @@ mod tests {
|
|||||||
HashMap::new(),
|
HashMap::new(),
|
||||||
0.0,
|
0.0,
|
||||||
values.to_vec(),
|
values.to_vec(),
|
||||||
|
HashMap::new(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -240,6 +240,7 @@ mod tests {
|
|||||||
HashMap::new(),
|
HashMap::new(),
|
||||||
0.0,
|
0.0,
|
||||||
values.to_vec(),
|
values.to_vec(),
|
||||||
|
HashMap::new(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -311,6 +311,7 @@ mod tests {
|
|||||||
HashMap::new(),
|
HashMap::new(),
|
||||||
value,
|
value,
|
||||||
intermediate_values,
|
intermediate_values,
|
||||||
|
HashMap::new(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+19
-1
@@ -9,6 +9,7 @@ use std::collections::HashMap;
|
|||||||
use crate::distribution::Distribution;
|
use crate::distribution::Distribution;
|
||||||
use crate::param::ParamValue;
|
use crate::param::ParamValue;
|
||||||
use crate::parameter::{ParamId, Parameter};
|
use crate::parameter::{ParamId, Parameter};
|
||||||
|
use crate::trial::AttrValue;
|
||||||
use crate::types::TrialState;
|
use crate::types::TrialState;
|
||||||
|
|
||||||
/// A completed trial with its parameters, distributions, and objective value.
|
/// A completed trial with its parameters, distributions, and objective value.
|
||||||
@@ -32,6 +33,8 @@ pub struct CompletedTrial<V = f64> {
|
|||||||
pub intermediate_values: Vec<(u64, f64)>,
|
pub intermediate_values: Vec<(u64, f64)>,
|
||||||
/// The state of the trial (Complete, Pruned, or Failed).
|
/// The state of the trial (Complete, Pruned, or Failed).
|
||||||
pub state: TrialState,
|
pub state: TrialState,
|
||||||
|
/// User-defined attributes stored during the trial.
|
||||||
|
pub user_attrs: HashMap<String, AttrValue>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<V> CompletedTrial<V> {
|
impl<V> CompletedTrial<V> {
|
||||||
@@ -51,10 +54,11 @@ impl<V> CompletedTrial<V> {
|
|||||||
value,
|
value,
|
||||||
intermediate_values: Vec::new(),
|
intermediate_values: Vec::new(),
|
||||||
state: TrialState::Complete,
|
state: TrialState::Complete,
|
||||||
|
user_attrs: HashMap::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Creates a new completed trial with intermediate values.
|
/// Creates a new completed trial with intermediate values and user attributes.
|
||||||
pub fn with_intermediate_values(
|
pub fn with_intermediate_values(
|
||||||
id: u64,
|
id: u64,
|
||||||
params: HashMap<ParamId, ParamValue>,
|
params: HashMap<ParamId, ParamValue>,
|
||||||
@@ -62,6 +66,7 @@ impl<V> CompletedTrial<V> {
|
|||||||
param_labels: HashMap<ParamId, String>,
|
param_labels: HashMap<ParamId, String>,
|
||||||
value: V,
|
value: V,
|
||||||
intermediate_values: Vec<(u64, f64)>,
|
intermediate_values: Vec<(u64, f64)>,
|
||||||
|
user_attrs: HashMap<String, AttrValue>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
id,
|
id,
|
||||||
@@ -71,6 +76,7 @@ impl<V> CompletedTrial<V> {
|
|||||||
value,
|
value,
|
||||||
intermediate_values,
|
intermediate_values,
|
||||||
state: TrialState::Complete,
|
state: TrialState::Complete,
|
||||||
|
user_attrs,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -114,6 +120,18 @@ impl<V> CompletedTrial<V> {
|
|||||||
.expect("parameter type mismatch: stored value incompatible with parameter")
|
.expect("parameter type mismatch: stored value incompatible with parameter")
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Gets a user attribute by key.
|
||||||
|
#[must_use]
|
||||||
|
pub fn user_attr(&self, key: &str) -> Option<&AttrValue> {
|
||||||
|
self.user_attrs.get(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns all user attributes.
|
||||||
|
#[must_use]
|
||||||
|
pub fn user_attrs(&self) -> &HashMap<String, AttrValue> {
|
||||||
|
&self.user_attrs
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A pending (running) trial with its parameters and distributions, but no objective value yet.
|
/// A pending (running) trial with its parameters and distributions, but no objective value yet.
|
||||||
|
|||||||
@@ -363,6 +363,7 @@ where
|
|||||||
trial.param_labels().clone(),
|
trial.param_labels().clone(),
|
||||||
value,
|
value,
|
||||||
trial.intermediate_values().to_vec(),
|
trial.intermediate_values().to_vec(),
|
||||||
|
trial.user_attrs().clone(),
|
||||||
);
|
);
|
||||||
completed.state = TrialState::Complete;
|
completed.state = TrialState::Complete;
|
||||||
self.completed_trials.write().push(completed);
|
self.completed_trials.write().push(completed);
|
||||||
@@ -471,6 +472,7 @@ where
|
|||||||
trial.param_labels().clone(),
|
trial.param_labels().clone(),
|
||||||
V::default(),
|
V::default(),
|
||||||
trial.intermediate_values().to_vec(),
|
trial.intermediate_values().to_vec(),
|
||||||
|
trial.user_attrs().clone(),
|
||||||
);
|
);
|
||||||
completed.state = TrialState::Pruned;
|
completed.state = TrialState::Pruned;
|
||||||
self.completed_trials.write().push(completed);
|
self.completed_trials.write().push(completed);
|
||||||
|
|||||||
@@ -13,6 +13,49 @@ use crate::pruner::Pruner;
|
|||||||
use crate::sampler::{CompletedTrial, Sampler};
|
use crate::sampler::{CompletedTrial, Sampler};
|
||||||
use crate::types::TrialState;
|
use crate::types::TrialState;
|
||||||
|
|
||||||
|
/// A user attribute value that can be stored on a trial.
|
||||||
|
#[derive(Clone, Debug, PartialEq)]
|
||||||
|
pub enum AttrValue {
|
||||||
|
/// A floating-point attribute.
|
||||||
|
Float(f64),
|
||||||
|
/// An integer attribute.
|
||||||
|
Int(i64),
|
||||||
|
/// A string attribute.
|
||||||
|
String(String),
|
||||||
|
/// A boolean attribute.
|
||||||
|
Bool(bool),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<f64> for AttrValue {
|
||||||
|
fn from(v: f64) -> Self {
|
||||||
|
Self::Float(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<i64> for AttrValue {
|
||||||
|
fn from(v: i64) -> Self {
|
||||||
|
Self::Int(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<String> for AttrValue {
|
||||||
|
fn from(v: String) -> Self {
|
||||||
|
Self::String(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<&str> for AttrValue {
|
||||||
|
fn from(v: &str) -> Self {
|
||||||
|
Self::String(v.to_owned())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<bool> for AttrValue {
|
||||||
|
fn from(v: bool) -> Self {
|
||||||
|
Self::Bool(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// A trial represents a single evaluation of the objective function.
|
/// A trial represents a single evaluation of the objective function.
|
||||||
///
|
///
|
||||||
/// Each trial has a unique ID and stores the sampled parameters along with
|
/// Each trial has a unique ID and stores the sampled parameters along with
|
||||||
@@ -41,6 +84,8 @@ pub struct Trial {
|
|||||||
intermediate_values: Vec<(u64, f64)>,
|
intermediate_values: Vec<(u64, f64)>,
|
||||||
/// The pruner used to decide whether to stop this trial early.
|
/// The pruner used to decide whether to stop this trial early.
|
||||||
pruner: Option<Arc<dyn Pruner>>,
|
pruner: Option<Arc<dyn Pruner>>,
|
||||||
|
/// User-defined attributes for logging, debugging, and analysis.
|
||||||
|
user_attrs: HashMap<String, AttrValue>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl core::fmt::Debug for Trial {
|
impl core::fmt::Debug for Trial {
|
||||||
@@ -55,6 +100,7 @@ impl core::fmt::Debug for Trial {
|
|||||||
.field("has_history", &self.history.is_some())
|
.field("has_history", &self.history.is_some())
|
||||||
.field("intermediate_values", &self.intermediate_values)
|
.field("intermediate_values", &self.intermediate_values)
|
||||||
.field("has_pruner", &self.pruner.is_some())
|
.field("has_pruner", &self.pruner.is_some())
|
||||||
|
.field("user_attrs", &self.user_attrs)
|
||||||
.finish()
|
.finish()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -92,6 +138,7 @@ impl Trial {
|
|||||||
history: None,
|
history: None,
|
||||||
intermediate_values: Vec::new(),
|
intermediate_values: Vec::new(),
|
||||||
pruner: None,
|
pruner: None,
|
||||||
|
user_attrs: HashMap::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -121,6 +168,7 @@ impl Trial {
|
|||||||
history: Some(history),
|
history: Some(history),
|
||||||
intermediate_values: Vec::new(),
|
intermediate_values: Vec::new(),
|
||||||
pruner: Some(pruner),
|
pruner: Some(pruner),
|
||||||
|
user_attrs: HashMap::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -209,6 +257,23 @@ impl Trial {
|
|||||||
&self.intermediate_values
|
&self.intermediate_values
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Sets a user attribute on this trial.
|
||||||
|
pub fn set_user_attr(&mut self, key: impl Into<String>, value: impl Into<AttrValue>) {
|
||||||
|
self.user_attrs.insert(key.into(), value.into());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Gets a user attribute by key.
|
||||||
|
#[must_use]
|
||||||
|
pub fn user_attr(&self, key: &str) -> Option<&AttrValue> {
|
||||||
|
self.user_attrs.get(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns all user attributes.
|
||||||
|
#[must_use]
|
||||||
|
pub fn user_attrs(&self) -> &HashMap<String, AttrValue> {
|
||||||
|
&self.user_attrs
|
||||||
|
}
|
||||||
|
|
||||||
/// Sets the trial state to Complete.
|
/// Sets the trial state to Complete.
|
||||||
pub(crate) fn set_complete(&mut self) {
|
pub(crate) fn set_complete(&mut self) {
|
||||||
self.state = TrialState::Complete;
|
self.state = TrialState::Complete;
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ fn trial_with_values(id: u64, intermediate_values: Vec<(u64, f64)>) -> Completed
|
|||||||
HashMap::new(),
|
HashMap::new(),
|
||||||
0.0,
|
0.0,
|
||||||
intermediate_values,
|
intermediate_values,
|
||||||
|
HashMap::new(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,148 @@
|
|||||||
|
use optimizer::parameter::{FloatParam, Parameter};
|
||||||
|
use optimizer::{AttrValue, Direction, Study};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn set_and_get_float_attr() {
|
||||||
|
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||||
|
let x = FloatParam::new(0.0, 1.0);
|
||||||
|
|
||||||
|
study
|
||||||
|
.optimize(1, |trial| {
|
||||||
|
let _ = x.suggest(trial)?;
|
||||||
|
trial.set_user_attr("score", 42.5);
|
||||||
|
assert_eq!(trial.user_attr("score"), Some(&AttrValue::Float(42.5)));
|
||||||
|
Ok::<_, optimizer::Error>(1.0)
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn set_and_get_int_attr() {
|
||||||
|
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||||
|
let x = FloatParam::new(0.0, 1.0);
|
||||||
|
|
||||||
|
study
|
||||||
|
.optimize(1, |trial| {
|
||||||
|
let _ = x.suggest(trial)?;
|
||||||
|
trial.set_user_attr("epoch", 42_i64);
|
||||||
|
assert_eq!(trial.user_attr("epoch"), Some(&AttrValue::Int(42)));
|
||||||
|
Ok::<_, optimizer::Error>(1.0)
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn set_and_get_string_attr() {
|
||||||
|
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||||
|
let x = FloatParam::new(0.0, 1.0);
|
||||||
|
|
||||||
|
study
|
||||||
|
.optimize(1, |trial| {
|
||||||
|
let _ = x.suggest(trial)?;
|
||||||
|
trial.set_user_attr("model", "resnet50");
|
||||||
|
assert_eq!(
|
||||||
|
trial.user_attr("model"),
|
||||||
|
Some(&AttrValue::String("resnet50".to_owned()))
|
||||||
|
);
|
||||||
|
Ok::<_, optimizer::Error>(1.0)
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn set_and_get_bool_attr() {
|
||||||
|
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||||
|
let x = FloatParam::new(0.0, 1.0);
|
||||||
|
|
||||||
|
study
|
||||||
|
.optimize(1, |trial| {
|
||||||
|
let _ = x.suggest(trial)?;
|
||||||
|
trial.set_user_attr("converged", true);
|
||||||
|
assert_eq!(trial.user_attr("converged"), Some(&AttrValue::Bool(true)));
|
||||||
|
Ok::<_, optimizer::Error>(1.0)
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn attrs_propagate_to_completed_trial() {
|
||||||
|
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||||
|
let x = FloatParam::new(0.0, 1.0);
|
||||||
|
|
||||||
|
study
|
||||||
|
.optimize(1, |trial| {
|
||||||
|
let _ = x.suggest(trial)?;
|
||||||
|
trial.set_user_attr("time_secs", 1.5);
|
||||||
|
trial.set_user_attr("tag", "baseline");
|
||||||
|
Ok::<_, optimizer::Error>(1.0)
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let best = study.best_trial().unwrap();
|
||||||
|
assert_eq!(best.user_attr("time_secs"), Some(&AttrValue::Float(1.5)));
|
||||||
|
assert_eq!(
|
||||||
|
best.user_attr("tag"),
|
||||||
|
Some(&AttrValue::String("baseline".to_owned()))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn overwrite_attr_replaces_value() {
|
||||||
|
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||||
|
let x = FloatParam::new(0.0, 1.0);
|
||||||
|
|
||||||
|
study
|
||||||
|
.optimize(1, |trial| {
|
||||||
|
let _ = x.suggest(trial)?;
|
||||||
|
trial.set_user_attr("key", "old");
|
||||||
|
trial.set_user_attr("key", "new");
|
||||||
|
assert_eq!(
|
||||||
|
trial.user_attr("key"),
|
||||||
|
Some(&AttrValue::String("new".to_owned()))
|
||||||
|
);
|
||||||
|
Ok::<_, optimizer::Error>(1.0)
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let best = study.best_trial().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
best.user_attr("key"),
|
||||||
|
Some(&AttrValue::String("new".to_owned()))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn missing_attr_returns_none() {
|
||||||
|
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||||
|
let x = FloatParam::new(0.0, 1.0);
|
||||||
|
|
||||||
|
study
|
||||||
|
.optimize(1, |trial| {
|
||||||
|
let _ = x.suggest(trial)?;
|
||||||
|
assert_eq!(trial.user_attr("nonexistent"), None);
|
||||||
|
Ok::<_, optimizer::Error>(1.0)
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let best = study.best_trial().unwrap();
|
||||||
|
assert_eq!(best.user_attr("nonexistent"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn user_attrs_map_returns_all() {
|
||||||
|
let study: Study<f64> = Study::new(Direction::Minimize);
|
||||||
|
let x = FloatParam::new(0.0, 1.0);
|
||||||
|
|
||||||
|
study
|
||||||
|
.optimize(1, |trial| {
|
||||||
|
let _ = x.suggest(trial)?;
|
||||||
|
trial.set_user_attr("a", 1.0);
|
||||||
|
trial.set_user_attr("b", true);
|
||||||
|
assert_eq!(trial.user_attrs().len(), 2);
|
||||||
|
Ok::<_, optimizer::Error>(1.0)
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let best = study.best_trial().unwrap();
|
||||||
|
assert_eq!(best.user_attrs().len(), 2);
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user