From 3e92f8bf92d382141796ef8d134a1bbd70aff401 Mon Sep 17 00:00:00 2001 From: tachsin Date: Thu, 1 Oct 2026 04:10:28 +0300 Subject: [PATCH 01/13] feat(math): portable erf, erfc and the scaled erfcx --- src/math.rs | 131 +++++++++++++++++++++++++++ tests/reference/special_functions.py | 38 ++++++++ 2 files changed, 169 insertions(+) create mode 100644 tests/reference/special_functions.py diff --git a/src/math.rs b/src/math.rs index 97cb9630..dc7988d7 100644 --- a/src/math.rs +++ b/src/math.rs @@ -258,6 +258,13 @@ from_libm! { cbrt = cbrt(x); /// `sqrt(x² + y²)` without overflow or underflow in between, the same on every platform. hypot = hypot(x, y); + /// The error function `erf(x) = 2/√π ∫₀ˣ e^(−t²) dt`, the same on every platform, within + /// 1 ulp. + erf = erf(x); + /// The complementary error function `erfc(x) = 1 − erf(x)`, accurate where `1 − erf(x)` + /// would cancel (large `x`), the same on every platform. It underflows to 0 above about + /// 26.5; [`erfcx`] keeps its scale. + erfc = erfc(x); } /// `x` to the integer power `n`, the same on every platform. @@ -292,6 +299,65 @@ pub fn sin_cos(x: f64) -> (f64, f64) { libm::sincos(x) } +/// The scaled complementary error function `erfcx(x) = e^(x²) erfc(x)`, the same on every +/// platform: finite where `erfc` underflows, about `1 / (x √π)` for large `x`. For logarithms of +/// normal tail probabilities, e.g. `ln Φ(z) = ln(erfcx(−z/√2) / 2) − z²/2` for very negative `z`. +/// +/// Up to 26 (where `erfc` is still a normal number) it's `e^(x²) · erfc(x)`, with `x²` split +/// exactly into two parts so that `e^(x²)` is as accurate as `exp`; above, the first 9 terms of +/// the asymptotic series `1 / (x √π) · Σ (−1)ⁿ (2n − 1)!! / (2x²)ⁿ` (Abramowitz and Stegun, +/// 1964, 7.1.23), whose remainder there is below 1e-19. Below 0, `2 e^(x²) − erfcx(−x)`, which +/// overflows to infinity below about −26.6. Within a few ulps everywhere. +#[must_use] +pub fn erfcx(x: f64) -> f64 { + if x.is_nan() { + return x; + } + if x < 0.0 { + return 2.0 * exp_square(x) - erfcx(-x); + } + if x < 26.0 { + return exp_square(x) * erfc(x); + } + // the asymptotic series in t = 1 / (2x²), by Horner's rule: 1 − t + 3t² − 15t³ + ...; 1 / x + // first, so that x² can't overflow + let inverse = 1.0 / x; + let t = 0.5 * inverse * inverse; + const DOUBLE_FACTORIALS: [f64; 9] = [ + 1.0, + 1.0, + 3.0, + 15.0, + 105.0, + 945.0, + 10_395.0, + 135_135.0, + 2_027_025.0, + ]; + let mut sum = 0.0; + for (n, factorial) in DOUBLE_FACTORIALS.iter().enumerate().rev() { + let term = if n % 2 == 0 { *factorial } else { -factorial }; + sum = sum * t + term; + } + inverse * std::f64::consts::FRAC_2_SQRT_PI * 0.5 * sum +} + +// e^(x²), with x² split exactly into hi + lo (Dekker's product), so that the rounding of x² +// doesn't cost accuracy: e^(hi + lo) = e^hi (1 + lo) to within lo², far below an ulp +fn exp_square(x: f64) -> f64 { + const SPLIT: f64 = 134_217_729.0; // 2^27 + 1 + let c = SPLIT * x; + let high = c - (c - x); + let low = x - high; + let square = x * x; + let error = ((high * high - square) + 2.0 * high * low) + low * low; + let e = exp(square); + if e.is_infinite() { + return e; + } + e + e * error +} + #[cfg(test)] mod tests { use super::*; @@ -406,6 +472,71 @@ mod tests { ); } + /// `erfcx` against tests/reference/special_functions.py (mpmath, 50 digits), on both sides of + /// each of its branches: 0, 26, and the overflow below −26.6. + #[test] + fn erfcx_values() { + let reference = [ + (-26.0, 7.657724931490568e+293), + (-10.0, 5.376234283632271e+43), + (-3.0, 16205.988853999586), + (-1.0, 5.008980080762283), + (-0.5, 1.952360489182557), + (-0.001, 1.0011293799198486), + (0.0, 1.0), + (1e-10, 0.999999999887162), + (0.25, 0.7703465477309968), + (0.5, 0.6156903441929259), + (1.0, 0.427583576155807), + (2.0, 0.25539567631050575), + (3.5, 0.1552936556088943), + (5.0, 0.11070463773306863), + (10.0, 0.05614099274382259), + (20.0, 0.02817434874105132), + (25.9, 0.021767181150738214), + (26.0, 0.021683584850562907), + (26.1, 0.021600627726346206), + (30.0, 0.01879588886141675), + (50.0, 0.011281536265323773), + (100.0, 0.005641613782989433), + (1000.0, 0.0005641893014533876), + (100000000.0, 5.641895835477562e-09), + (1e+20, 5.6418958354775626e-21), + (1e+200, 5.641895835477563e-201), + ]; + for (x, expected) in reference { + let value = erfcx(x); + assert!( + ulps(value, expected) <= 4, + "erfcx({x}) = {value}, not {expected}" + ); + } + assert_eq!(erfcx(-27.0), f64::INFINITY); + assert_eq!(erfcx(f64::INFINITY), 0.0); + assert!(erfcx(f64::NAN).is_nan()); + // continuous across 26, and decreasing + let mut previous = erfcx(0.0); + let mut rng = StreamRng::seed_from_u64(2); + let mut points: Vec = (0..2_000).map(|_| rng.unit_f64() * 60.0).collect(); + points.sort_by(f64::total_cmp); + for x in points { + let value = erfcx(x); + assert!(value <= previous, "erfcx({x})"); + previous = value; + } + assert_eq!( + [erf(0.5), erfc(0.5), erfcx(0.7), erfcx(-4.2), erfcx(31.0)].map(f64::to_bits), + [ + 4602863465656806866, // 0.5204998778130465 + 4602309526204327004, // 0.4795001221869535 + 4602912378887895979, // 0.525930337349441 + 4725920843874906963, // 91618811.94453172 + 4580900192363489479, // 0.018190209599233478 + ], + "the portable math changed, which breaks reproducibility" + ); + } + /// The public functions from libm: fixed bits, on every platform, and within 1 ulp of std. #[test] fn libm_values() { diff --git a/tests/reference/special_functions.py b/tests/reference/special_functions.py new file mode 100644 index 00000000..d9d64d95 --- /dev/null +++ b/tests/reference/special_functions.py @@ -0,0 +1,38 @@ +"""Reference values of the special functions in genoxide::math and of the acquisition functions, +to 50 significant digits with mpmath, independently of genoxide. The tests embed its output, +rounded to the nearest f64 (repr), with the inputs they come from. + + pip install mpmath + python tests/reference/special_functions.py +""" + +from mpmath import erfc, exp, log, mp, mpf, ncdf, npdf + +mp.dps = 50 + +ERFCX = [-26.0, -10.0, -3.0, -1.0, -0.5, -1e-3, 0.0, 1e-10, 0.25, 0.5, 1.0, 2.0, 3.5, 5.0, + 10.0, 20.0, 25.9, 26.0, 26.1, 30.0, 50.0, 100.0, 1e3, 1e8, 1e20, 1e200] +LOG_H = [5.0, 1.0, 0.0, -0.5, -0.999, -1.0, -1.001, -2.0, -5.0, -10.0, -30.0, -40.0, -1e3, + -1e7, -6.7e7, -6.72e7, -1e8, -1e10, -1e100] + + +def erfcx(x): + if x > 1e10: + # mpmath's erfc overflows here; the asymptotic series' first terms are exact to far below + # an f64's precision + t = 1 / (2 * x * x) + return (1 - t + 3 * t * t) / (x * mp.sqrt(mp.pi)) + return exp(x * x) * erfc(x) + + +def h(z): + # Ament et al. (2023): EI = sigma h(z), z = (mu - best) / sigma, maximizing + return npdf(z) + z * ncdf(z) + + +print("// erfcx(x)") +for x in ERFCX: + print(f"({x!r}, {float(erfcx(mpf(x)))!r}),") +print("// ln h(z)") +for z in LOG_H: + print(f"({z!r}, {float(log(h(mpf(z))))!r}),") From b56959476efd99b4ee049354057f5cd350041b71 Mon Sep 17 00:00:00 2001 From: tachsin Date: Thu, 1 Oct 2026 04:10:28 +0300 Subject: [PATCH 02/13] feat(genome): Latin hypercube samples of real genomes --- src/genome/real.rs | 61 +++++++++++++++++++++++ tests/latin_hypercube.rs | 102 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 tests/latin_hypercube.rs diff --git a/src/genome/real.rs b/src/genome/real.rs index 6edc926e..0e519ec2 100644 --- a/src/genome/real.rs +++ b/src/genome/real.rs @@ -187,6 +187,67 @@ impl Real { pub(crate) fn variable_genes(&self) -> &[usize] { &self.variable } + + /// `n` genomes of a Latin hypercube sample (McKay, Beckman and Conover, 1979): each gene's + /// range is cut into `n` equal strata, and every stratum of every gene holds exactly one of the + /// genomes, at a uniform random point inside it. The strata of the genes are matched by an + /// independent random permutation per gene. The genomes cover each gene's range far more + /// evenly than `n` random genomes do, which is why it's the usual initial design of a surrogate + /// model; it's also a starting population for any algorithm, through `initial_genomes`. + /// + /// A gene whose bounds are equal takes its single value. The random numbers are drawn gene by + /// gene: the permutation (Fisher-Yates, from the last stratum down), then a uniform number per + /// genome, so a seed gives the same sample on every platform. + /// + /// ``` + /// use genoxide::prelude::*; + /// + /// let real = Real::uniform(3, 0.0..=1.0)?; + /// let sample = real.latin_hypercube(10, &mut StreamRng::seed_from_u64(1))?; + /// // every tenth of every gene's range holds exactly one genome + /// for gene in 0..3 { + /// let mut strata: Vec = sample.iter().map(|x| (x[gene] * 10.0) as usize).collect(); + /// strata.sort(); + /// assert_eq!(strata, (0..10).collect::>()); + /// } + /// # Ok::<(), genoxide::Error>(()) + /// ``` + /// + /// # Errors + /// + /// [`Error::InvalidSetting`] for `n` of 0 or above 2^24. + pub fn latin_hypercube(&self, n: usize, rng: &mut StreamRng) -> Result> { + if n == 0 { + return Err(Error::InvalidSetting { + setting: "n", + reason: "a Latin hypercube needs at least 1 genome".to_string(), + }); + } + crate::operator::check_size("n", n)?; + let mut genomes = vec![Vec::with_capacity(self.bounds.len()); n]; + let mut strata: Vec = Vec::with_capacity(n); + for range in &self.bounds { + let (low, high) = (*range.start(), *range.end()); + if low == high { + for genome in &mut genomes { + genome.push(low); + } + continue; + } + strata.clear(); + strata.extend(0..n); + for last in (1..n).rev() { + let other = rng.below(last + 1); + strata.swap(last, other); + } + let width = high - low; + for (genome, &stratum) in genomes.iter_mut().zip(&strata) { + let u = (stratum as f64 + rng.unit_f64()) / n as f64; + genome.push((low + u * width).clamp(low, high)); + } + } + Ok(genomes.into_iter().map(Reals::from).collect()) + } } // a uniformly random value in `range`, which is finite diff --git a/tests/latin_hypercube.rs b/tests/latin_hypercube.rs new file mode 100644 index 00000000..bfb58862 --- /dev/null +++ b/tests/latin_hypercube.rs @@ -0,0 +1,102 @@ +//! Latin hypercube samples of real genomes: one genome per stratum of every gene. + +use genoxide::prelude::*; + +#[test] +fn every_stratum_of_every_gene_holds_one_genome() { + let real = Real::new([-5.0..=5.0, 0.0..=1.0, 100.0..=1e6, -1e-3..=0.0]).unwrap(); + for (n, seed) in [(1, 1), (2, 2), (7, 3), (100, 4), (1_000, 5)] { + let sample = real + .latin_hypercube(n, &mut StreamRng::seed_from_u64(seed)) + .unwrap(); + assert_eq!(sample.len(), n); + for (gene, range) in real.bounds().iter().enumerate() { + let (low, high) = (*range.start(), *range.end()); + let mut strata: Vec = sample + .iter() + .map(|genome| { + let x = genome[gene]; + assert!((low..=high).contains(&x)); + // the stratum, with a point on a boundary in the one below + let position = (x - low) / (high - low) * n as f64; + (position as usize).min(n - 1) + }) + .collect(); + strata.sort(); + assert_eq!(strata, (0..n).collect::>(), "n = {n}, gene {gene}"); + } + } +} + +#[test] +fn fixed_genes_take_their_value() { + let real = Real::new([2.5..=2.5, 0.0..=1.0]).unwrap(); + let sample = real + .latin_hypercube(20, &mut StreamRng::seed_from_u64(6)) + .unwrap(); + assert!(sample.iter().all(|genome| genome[0] == 2.5)); +} + +#[test] +fn a_seed_gives_the_same_sample_on_every_platform() { + let real = Real::uniform(3, -1.0..=1.0).unwrap(); + let sample = |seed| { + real.latin_hypercube(4, &mut StreamRng::seed_from_u64(seed)) + .unwrap() + }; + assert_eq!(sample(7), sample(7)); + assert_ne!(sample(7), sample(8)); + let bits: Vec = sample(7)[0].iter().map(|x| x.to_bits()).collect(); + assert_eq!( + bits, + [ + 4600217523584051144, + 13829652665615299316, + 4585620556182390272 + ], + "the sample changed, which breaks reproducibility" + ); +} + +#[test] +fn a_sample_of_none_or_too_many_is_an_error() { + let real = Real::uniform(2, 0.0..=1.0).unwrap(); + let mut rng = StreamRng::seed_from_u64(1); + for n in [0, (1 << 24) + 1] { + assert!(matches!( + real.latin_hypercube(n, &mut rng), + Err(Error::InvalidSetting { setting: "n", .. }) + )); + } +} + +#[test] +fn it_seeds_an_algorithm() { + // a Latin hypercube as a GA's initial population + let real = Real::uniform(5, -5.12..=5.12).unwrap(); + let initial = real + .latin_hypercube(40, &mut StreamRng::seed_from_u64(9)) + .unwrap(); + let ga = Ga::builder(real) + .population_size(40) + .initial_genomes(initial.clone()) + .select(Tournament::new(3).unwrap()) + .crossover(SimulatedBinaryCrossover::new(15.0).unwrap()) + .mutate(PolynomialMutation::per_gene(0.2, 20.0).unwrap()) + .minimize() + .seed(9) + .build() + .unwrap(); + let mut engine = Engine::new(ga, |x: &Reals| x.iter().map(|xi| xi * xi).sum::()) + .stop_when(Stop::generations(0)); + engine.run().unwrap(); + let population: Vec<&Reals> = engine + .algorithm() + .population() + .iter() + .map(Individual::genome) + .collect(); + for genome in &initial { + assert!(population.contains(&genome)); + } +} From e162f32814ef5704f298de81c04dcd18a1e193b0 Mon Sep 17 00:00:00 2001 From: tachsin Date: Thu, 1 Oct 2026 04:10:28 +0300 Subject: [PATCH 03/13] feat(bo): expected improvement, log-EI, probability of improvement and the upper confidence bound --- src/algorithm.rs | 1 + src/algorithm/bo.rs | 7 + src/algorithm/bo/acquisition.rs | 363 ++++++++++++++++++++++++++++++++ 3 files changed, 371 insertions(+) create mode 100644 src/algorithm/bo.rs create mode 100644 src/algorithm/bo/acquisition.rs diff --git a/src/algorithm.rs b/src/algorithm.rs index 3b917fc1..7d5b11fa 100644 --- a/src/algorithm.rs +++ b/src/algorithm.rs @@ -27,6 +27,7 @@ //! # Ok::<(), genoxide::Error>(()) //! ``` +pub mod bo; pub mod cmaes; pub mod continuation; pub mod de; diff --git a/src/algorithm/bo.rs b/src/algorithm/bo.rs new file mode 100644 index 00000000..3b48c515 --- /dev/null +++ b/src/algorithm/bo.rs @@ -0,0 +1,7 @@ +//! Bayesian optimization: a surrogate model of an expensive function, and acquisition functions +//! that pick where to evaluate it next. +//! +//! [`acquisition`] has the acquisition functions, from a model's predictive mean and standard +//! deviation at a point: they work with any surrogate model that gives those. + +pub mod acquisition; diff --git a/src/algorithm/bo/acquisition.rs b/src/algorithm/bo/acquisition.rs new file mode 100644 index 00000000..822e825a --- /dev/null +++ b/src/algorithm/bo/acquisition.rs @@ -0,0 +1,363 @@ +//! Acquisition functions: how much a point is worth evaluating, from a surrogate model's +//! predictive mean `μ` and standard deviation `σ` there, and the best value observed so far. +//! +//! Every function here is higher for a point more worth evaluating, for both objectives: when +//! minimizing, an improvement is a value below the best, and the functions mirror themselves. With +//! `z = (μ − best) / σ` when maximizing (`(best − μ) / σ` when minimizing), the standardized +//! improvement: +//! +//! | Function | Value | Reference | +//! |---|---|---| +//! | [`expected_improvement`] | `σ (φ(z) + z Φ(z))`, the expected amount by which the point beats the best | Močkus (1975); Jones, Schonlau and Welch (1998) | +//! | [`log_expected_improvement`] | its logarithm, finite and smooth where the expected improvement underflows to 0 | Ament, Daulton, Eriksson, Balandat and Bakshy (2023) | +//! | [`probability_of_improvement`] | `Φ(z − ξ / σ)`, the probability that the point beats the best by `ξ` | Kushner (1964) | +//! | [`upper_confidence_bound`] | `μ + √β σ` (`−μ + √β σ` when minimizing), an optimistic value | Srinivas, Krause, Kakade and Seeger (2010) | +//! +//! `φ` and `Φ` are the standard normal density and distribution function. With `σ = 0` (a +//! point the model is sure of) they take their limits: the improvement itself, or 0. A negative +//! or NaN `σ`, or a NaN mean or best, gives NaN. Every function uses [`math`](crate::math)'s +//! portable functions only, so it gives the same bits on every platform. +//! +//! ``` +//! use genoxide::Objective; +//! use genoxide::algorithm::bo::acquisition::{expected_improvement, log_expected_improvement}; +//! +//! // a model that predicts 1.0 ± 0.5 where the best so far is 1.2, minimizing +//! let ei = expected_improvement(1.0, 0.5, 1.2, Objective::Minimize); +//! assert!((log_expected_improvement(1.0, 0.5, 1.2, Objective::Minimize) - ei.ln()).abs() < 1e-14); +//! // far from any improvement, the expected improvement is 0, but its logarithm still ranks +//! // points +//! assert_eq!(expected_improvement(100.0, 0.5, 1.2, Objective::Minimize), 0.0); +//! let far = log_expected_improvement(100.0, 0.5, 1.2, Objective::Minimize); +//! let farther = log_expected_improvement(200.0, 0.5, 1.2, Objective::Minimize); +//! assert!(far.is_finite() && farther < far); +//! ``` +//! +//! References: Močkus, J. (1975). On Bayesian methods for seeking the extremum. *Optimization +//! Techniques IFIP Technical Conference 1974*, LNCS 27: 400-404. Jones, D. R., Schonlau, M. and +//! Welch, W. J. (1998). Efficient global optimization of expensive black-box functions. *Journal of +//! Global Optimization* 13(4): 455-492. Ament, S., Daulton, S., Eriksson, D., Balandat, M. and +//! Bakshy, E. (2023). Unexpected improvements to expected improvement for Bayesian optimization. +//! *NeurIPS 2023*, arXiv:2310.20708. Kushner, H. J. (1964). A new method of locating the maximum +//! point of an arbitrary multipeak curve in the presence of noise. *Journal of Basic Engineering* +//! 86(1): 97-106. Srinivas, N., Krause, A., Kakade, S. and Seeger, M. (2010). Gaussian process +//! optimization in the bandit setting: no regret and experimental design. *ICML 2010*, +//! arXiv:0912.3995. + +use crate::Objective; +use crate::math::{erfc, erfcx, exp, exp_m1, ln, ln_1p}; +use std::f64::consts::{FRAC_1_SQRT_2, LN_2}; + +/// The expected improvement of a point over `best`: `σ (φ(z) + z Φ(z))`, at least 0. It +/// underflows to 0 when `z` is below about −38; [`log_expected_improvement`] doesn't. +/// +/// With `sd = 0`, the improvement itself: `max(μ − best, 0)` when maximizing. +#[must_use] +pub fn expected_improvement(mean: f64, sd: f64, best: f64, objective: Objective) -> f64 { + let improvement = improvement(mean, best, objective); + if invalid(sd) || improvement.is_nan() { + return f64::NAN; + } + if sd == 0.0 { + return improvement.max(0.0); + } + let z = improvement / sd; + (sd * (normal_pdf(z) + z * normal_cdf(z))).max(0.0) +} + +/// The natural logarithm of the [expected improvement](expected_improvement), computed so that it +/// stays finite, accurate and smooth where the expected improvement underflows to 0: Ament et +/// al.'s (2023) `LogEI`, `ln σ + log_h(z)` (their eq. 8), with `log_h` as their eq. 9. +/// +/// For `z > −1`, `ln(φ(z) + z Φ(z))` directly. For `−1/√ε < z ≤ −1` (ε the machine epsilon), +/// `−z²/2 − c₁ + log1mexp(ln(erfcx(−z/√2) |z|) + c₂)`, with `c₁ = ln(2π)/2`, `c₂ = ln(π/2)/2` and +/// `log1mexp(x) = ln(1 − eˣ)` (their eq. 13). Below, the asymptote `−z²/2 − c₁ − 2 ln |z|`. +/// +/// With `sd = 0`, the logarithm of the improvement: `−∞` without one. +#[must_use] +pub fn log_expected_improvement(mean: f64, sd: f64, best: f64, objective: Objective) -> f64 { + let improvement = improvement(mean, best, objective); + if invalid(sd) || improvement.is_nan() { + return f64::NAN; + } + if sd == 0.0 { + return if improvement > 0.0 { + ln(improvement) + } else { + f64::NEG_INFINITY + }; + } + ln(sd) + log_h(improvement / sd) +} + +/// The probability that a point improves on `best` by more than `xi` (`ξ ≥ 0`, often 0.01 of the +/// values' range, to prefer larger improvements): `Φ(z − ξ / σ)`. +/// +/// With `sd = 0`, 1 if the improvement is larger than `xi`, else 0. NaN for a negative or NaN +/// `xi`. +#[must_use] +pub fn probability_of_improvement( + mean: f64, + sd: f64, + best: f64, + xi: f64, + objective: Objective, +) -> f64 { + let improvement = improvement(mean, best, objective) - xi; + if invalid(sd) || invalid(xi) || improvement.is_nan() { + return f64::NAN; + } + if sd == 0.0 { + return if improvement > 0.0 { 1.0 } else { 0.0 }; + } + normal_cdf(improvement / sd) +} + +/// The upper confidence bound `μ + √β σ` when maximizing, and the lower one, negated, `−μ + √β σ`, +/// when minimizing: higher is better either way. `β ≥ 0` weighs exploration (`σ`) against +/// exploitation (`μ`); Srinivas et al.'s schedule raises it slowly with the number of evaluations. +/// +/// NaN for a negative or NaN `sd` or `beta`. +#[must_use] +pub fn upper_confidence_bound(mean: f64, sd: f64, beta: f64, objective: Objective) -> f64 { + if invalid(sd) || invalid(beta) { + return f64::NAN; + } + let mean = match objective { + Objective::Maximize => mean, + Objective::Minimize => -mean, + }; + mean + beta.sqrt() * sd +} + +// a negative or NaN standard deviation or parameter +fn invalid(x: f64) -> bool { + x.is_nan() || x < 0.0 +} + +// how much `mean` beats `best`, positive for an improvement +fn improvement(mean: f64, best: f64, objective: Objective) -> f64 { + match objective { + Objective::Maximize => mean - best, + Objective::Minimize => best - mean, + } +} + +// ln(2π) / 2 and ln(π/2) / 2, Ament et al.'s c₁ and c₂ +const C1: f64 = 0.918_938_533_204_672_8; +const C2: f64 = 0.225_791_352_644_727_43; + +// 1 / √(2π) +const FRAC_1_SQRT_2PI: f64 = 0.398_942_280_401_432_7; + +// the standard normal density +fn normal_pdf(z: f64) -> f64 { + FRAC_1_SQRT_2PI * exp(-0.5 * z * z) +} + +// the standard normal distribution function, Φ(z) = erfc(−z / √2) / 2, accurate in both tails +fn normal_cdf(z: f64) -> f64 { + 0.5 * erfc(-z * FRAC_1_SQRT_2) +} + +// ln(φ(z) + z Φ(z)), Ament et al.'s eq. 9 +fn log_h(z: f64) -> f64 { + // 1 / √ε: below, erfcx(−z/√2) |z| √(π/2) rounds to 1, and the asymptote is exact + const ASYMPTOTE: f64 = 67_108_864.0; // 2^26 = 1 / √(2^-52) + if z > -1.0 { + ln(normal_pdf(z) + z * normal_cdf(z)) + } else if z > -ASYMPTOTE { + -0.5 * z * z - C1 + log1mexp(ln(erfcx(-z * FRAC_1_SQRT_2) * -z) + C2) + } else { + -0.5 * z * z - C1 - 2.0 * ln(-z) + } +} + +// ln(1 − eˣ) for x ≤ 0, accurate near 0 and for very negative x (Mächler, 2012, cited by Ament +// et al. as their eq. 13) +fn log1mexp(x: f64) -> f64 { + if x > -LN_2 { + ln(-exp_m1(x)) + } else { + ln_1p(-exp(x)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use Objective::{Maximize, Minimize}; + use std::f64::consts::PI; + + // ulps between a and b + fn ulps(a: f64, b: f64) -> u64 { + let key = |x: f64| { + let bits = x.to_bits() as i64; + if bits < 0 { i64::MIN - bits } else { bits } + }; + key(a).abs_diff(key(b)) + } + + #[test] + fn expected_improvement_closed_forms() { + // at μ = best and σ = 1, EI = φ(0) = 1 / √(2π) + let phi_0 = 0.398_942_280_401_432_7; + assert_eq!(phi_0, 1.0 / (2.0 * PI).sqrt()); + // correctly rounded, from mpmath: computed in f64, ln(2π) / 2 is 1 ulp below + assert_eq!(C1, 0.9189385332046728); + assert!(ulps(C1, ln(2.0 * PI) / 2.0) <= 1); + assert_eq!(C2, 0.22579135264472744); + assert!(ulps(C2, ln(PI / 2.0) / 2.0) <= 1); + assert_eq!(expected_improvement(3.0, 1.0, 3.0, Maximize), phi_0); + assert_eq!(expected_improvement(3.0, 1.0, 3.0, Minimize), phi_0); + // σ scales it + assert_eq!(expected_improvement(3.0, 2.0, 3.0, Maximize), 2.0 * phi_0); + // no uncertainty: the improvement itself + assert_eq!(expected_improvement(5.0, 0.0, 3.0, Maximize), 2.0); + assert_eq!(expected_improvement(5.0, 0.0, 3.0, Minimize), 0.0); + assert_eq!(expected_improvement(1.0, 0.0, 3.0, Minimize), 2.0); + // far better than the best: about the improvement + let sure = expected_improvement(13.0, 1.0, 3.0, Maximize); + assert!((sure - 10.0).abs() < 1e-12); + for invalid in [ + expected_improvement(1.0, -1.0, 0.0, Maximize), + expected_improvement(1.0, f64::NAN, 0.0, Maximize), + expected_improvement(f64::NAN, 1.0, 0.0, Maximize), + ] { + assert!(invalid.is_nan()); + } + } + + #[test] + fn minimizing_mirrors_maximizing() { + for (mean, sd, best) in [(0.3, 0.7, 1.0), (-2.0, 0.1, 5.0), (4.0, 3.0, -1.0)] { + assert_eq!( + expected_improvement(mean, sd, best, Maximize), + expected_improvement(-mean, sd, -best, Minimize) + ); + assert_eq!( + log_expected_improvement(mean, sd, best, Maximize), + log_expected_improvement(-mean, sd, -best, Minimize) + ); + assert_eq!( + probability_of_improvement(mean, sd, best, 0.01, Maximize), + probability_of_improvement(-mean, sd, -best, 0.01, Minimize) + ); + assert_eq!( + upper_confidence_bound(mean, sd, 2.0, Maximize), + upper_confidence_bound(-mean, sd, 2.0, Minimize) + ); + } + } + + #[test] + fn log_h_against_mpmath() { + // tests/reference/special_functions.py: ln(φ(z) + z Φ(z)) to 50 digits, rounded + let reference = [ + (5.0, 1.6094379231264313), + (1.0, 0.08002621884930694), + (0.0, -0.9189385332046728), + (-0.5, -1.6205162643873199), + (-0.999, -2.4832171154475855), + (-1.0, -2.4851210257126413), + (-1.001, -2.4870256579553893), + (-2.0, -4.768783523917114), + (-5.0, -16.74430116266099), + (-10.0, -55.55312203612235), + (-30.0, -457.724653760598), + (-40.0, -808.29856835662), + (-1000.0, -500014.73445209116), + (-10000000.0, -50000000000033.16), + (-67000000.0, -2244500000000037.0), + (-67200000.0, -2257920000000037.0), + (-100000000.0, -5000000000000038.0), + (-10000000000.0, -5e+19), + (-1e+100, -5e+199), + ]; + for (z, expected) in reference { + let value = log_h(z); + assert!( + ulps(value, expected) <= 8, + "log_h({z}) = {value}, not {expected}" + ); + } + } + + #[test] + fn log_expected_improvement_is_the_logarithm_where_it_can_be() { + for z in (-300..=300).map(|k| f64::from(k) / 10.0) { + for sd in [1e-3, 1.0, 50.0] { + let mean = 2.0 + z * sd; + let ei = expected_improvement(mean, sd, 2.0, Maximize); + let log_ei = log_expected_improvement(mean, sd, 2.0, Maximize); + assert!( + (log_ei - ln(ei)).abs() <= 1e-12 * ln(ei).abs().max(1.0), + "z = {z}, σ = {sd}: {log_ei} against ln {ei}" + ); + } + } + } + + #[test] + fn log_expected_improvement_keeps_ranking_where_it_underflows() { + let mut previous = f64::INFINITY; + for k in 0..400 { + let z = -40.0 * 1.06_f64.powi(k); + let log_ei = log_expected_improvement(z, 1.0, 0.0, Maximize); + assert_eq!(expected_improvement(z, 1.0, 0.0, Maximize), 0.0); + assert!(log_ei.is_finite() || z < -1e154, "z = {z}"); + assert!(log_ei < previous, "z = {z}"); + previous = log_ei; + } + assert_eq!( + log_expected_improvement(0.0, 0.0, 1.0, Maximize), + f64::NEG_INFINITY + ); + assert_eq!(log_expected_improvement(3.0, 0.0, 1.0, Maximize), LN_2); + } + + #[test] + fn probability_of_improvement_and_upper_confidence_bound() { + assert_eq!( + probability_of_improvement(1.0, 2.0, 1.0, 0.0, Maximize), + 0.5 + ); + // one standard deviation above the best + let one_sigma = probability_of_improvement(3.0, 2.0, 1.0, 0.0, Maximize); + assert!((one_sigma - 0.841_344_746_068_542_9).abs() < 1e-15); + // ξ asks for more + assert!(probability_of_improvement(3.0, 2.0, 1.0, 1.0, Maximize) < one_sigma); + assert_eq!( + probability_of_improvement(3.0, 0.0, 1.0, 1.0, Maximize), + 1.0 + ); + assert_eq!( + probability_of_improvement(3.0, 0.0, 1.0, 2.0, Maximize), + 0.0 + ); + assert!(probability_of_improvement(3.0, 1.0, 1.0, -1.0, Maximize).is_nan()); + assert_eq!(upper_confidence_bound(1.0, 2.0, 4.0, Maximize), 5.0); + assert_eq!(upper_confidence_bound(1.0, 2.0, 4.0, Minimize), 3.0); + assert!(upper_confidence_bound(1.0, 2.0, -1.0, Maximize).is_nan()); + } + + #[test] + fn portable_values() { + let values = [ + expected_improvement(0.3, 0.7, 1.0, Maximize), + log_expected_improvement(0.3, 0.7, 1.0, Maximize), + log_expected_improvement(-30.0, 0.7, 1.0, Maximize), + probability_of_improvement(0.3, 0.7, 1.0, 0.01, Maximize), + ]; + assert_eq!( + values.map(f64::to_bits), + [ + 4588565738335155431, // 0.05832082941138044 + 13836953611289785667, // -2.8417959696513733 + 13875286729078217892, // -989.4707096078387 + 4594760526153066046, // 0.15522321934831668 + ], + "the acquisition functions changed, which breaks reproducibility" + ); + } +} From d3b47de1b3195adafeca6c7508585a49dd5d994f Mon Sep 17 00:00:00 2001 From: tachsin Date: Thu, 1 Oct 2026 04:10:28 +0300 Subject: [PATCH 04/13] docs: the first part of batch B, and the sources read for it --- ROADMAP.md | 3 ++- docs/features.md | 2 ++ docs/optimization-plan.md | 12 ++++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/ROADMAP.md b/ROADMAP.md index 91590109..87e0695e 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -226,7 +226,8 @@ The plan: [docs/gp-neuroevolution-plan.md](docs/gp-neuroevolution-plan.md). - [x] Continuation in stages that keeps the optimizer's state, in Python (`Continuation`, `Continue`; batch A3) ### 0.13: Bayesian optimization -- [ ] Gaussian processes; EI, log-EI, UCB and PI; batch, constrained and integer-variable Bayesian optimization, also on the asynchronous engine (batch B) +- [x] EI, log-EI, UCB and PI; Latin hypercube designs; portable `erf`, `erfc` and `erfcx` (batch B, the parts that need neither linear algebra nor L-BFGS-B) +- [ ] Gaussian processes; batch, constrained and integer-variable Bayesian optimization, also on the asynchronous engine (batch B; after the linear algebra, [#373](https://github.com/tachsin/genoxide/issues/373), and L-BFGS-B) ### 0.14: Constrained nonlinear programming - [ ] SQP and the augmented Lagrangian, on the constrained test problems (batch C) diff --git a/docs/features.md b/docs/features.md index 1107bf02..c2e84935 100644 --- a/docs/features.md +++ b/docs/features.md @@ -17,6 +17,7 @@ What genoxide has on main; [docs.rs](https://docs.rs/genoxide) documents the lat - Integer and real, bounded per gene - Permutation - Real with a self-adaptive step size +- Latin hypercube samples of real genomes (`Real::latin_hypercube`, McKay et al. 1979), for initial designs and starting populations - Trees of a genetic program, strongly typed ## Genetic algorithms @@ -91,6 +92,7 @@ What genoxide has on main; [docs.rs](https://docs.rs/genoxide) documents the lat - **A fitness function that changes during a run:** the algorithms re-evaluate what they keep, e.g. after adapting penalty weights, without comparing old and new values, in Rust and Python. - **Extras with the fitness:** a fitness function returns what it computed along with the fitness (`Evaluated`), e.g. the terms of a penalty or a secondary measure; the engines keep it for the population and the best, for observers, the hall of fame and the outcome, without evaluating again and without changing the search. - **Gradients for gradient-based methods:** a fitness function supplies the gradient of its score along with it (`Differentiable` wraps a closure; a batch form takes a generation's gradients in one flat buffer), or an algorithm estimates it by forward or central differences, with the steps of Gill, Murray, Saunders and Wright, every point inside the bounds and fixed genes skipped, evaluated in one parallel or batch round; `gradient::check` tests a hand-written gradient against central differences. Algorithms ask for gradients only when they need them: the engine's path for every other algorithm is unchanged. +- **Acquisition functions** (`algorithm::bo::acquisition`), the building blocks of Bayesian optimization for any surrogate model that gives a mean and a standard deviation: expected improvement, its logarithm computed stably where it underflows (Ament et al. 2023's LogEI), the probability of improvement and the upper confidence bound. Portable `erf`, `erfc` and the scaled `erfcx` in `genoxide::math`. - **Constraints:** Deb's feasibility rules (a fitness function returns a score and a constraint violation), and penalty functions. A fitness function can also give each inequality constraint's value and the constraint Jacobian (`Constrained`, a closure), for the methods that use them, such as MMA, while every other algorithm takes it as a score and a violation. - **Cancellation**, and **checkpoints** to resume a run (`serde` feature). - **Observers:** statistics per generation, hall of fame, progress lines, `tracing`. diff --git a/docs/optimization-plan.md b/docs/optimization-plan.md index 9d743019..d637bef5 100644 --- a/docs/optimization-plan.md +++ b/docs/optimization-plan.md @@ -321,6 +321,18 @@ picks the next points, so that tens to a few hundred evaluations suffice. | TPE | Bergstra, J., Bardenet, R., Bengio, Y. and Kégl, B. (2011). Algorithms for hyper-parameter optimization. NeurIPS 24. Watanabe, S. (2023). Tree-structured Parzen estimator: understanding its algorithm components and their roles for better empirical performance. arXiv:2304.11127 | Cheaper than a GP per step (linear in the observations), any number of evaluations, integer and categorical genes natively; the multivariate kernel as in Falkner, S., Klein, A. and Hutter, F. (2018). BOHB. ICML 2018. arXiv:1807.01774 | E | | Trust-region BO (TuRBO-1, TuRBO-m) | Eriksson, D., Pearce, M., Gardner, J., Turner, R. and Poloczek, M. (2019). Scalable global optimization via local Bayesian optimization. NeurIPS 2019. arXiv:1910.01739 | Tens to a few hundred variables and thousands of evaluations: local GPs on the points in a trust region that grows on success and shrinks on failure. Thompson sampling needs a joint posterior sample at thousands of candidates (a Cholesky of that size): fewer candidates, or random Fourier features, decided in batch E | E | +**Batch B's first part, read on 2026-10-01** (the acquisition functions, `erfcx` and the Latin +hypercube, which need neither linear algebra nor L-BFGS-B). Ament et al. (2023): **verified**, +section 4.1 (eq. 8 and 9) and appendix A.1 (eq. 12 to 14): `LogEI = log_h(z) + ln σ` with +`z = (μ − y*)/σ` (maximizing), the three cases of `log_h` with their thresholds −1 and −1/√ε, the +constants c₁ = ln(2π)/2 and c₂ = ln(π/2)/2, and `log1mexp` as implemented. EI, PI and UCB are +their closed forms, not re-read from Močkus, Jones et al., Kushner and Srinivas et al.; the tests +check them against hand derivations (EI = φ(0) at μ = y*, σ = 1) and mpmath. McKay et al. (1979): +not re-read; the Latin hypercube is the standard construction (one point per stratum per gene, +independent permutations), which the tests check directly. `erfcx` is genoxide's own: libm's +`erfc` times an exactly split `e^(x²)` below 26, the asymptotic series of Abramowitz and Stegun +7.1.23 above, checked against mpmath (`tests/reference/special_functions.py`) within 4 ulps. + ### 1.6 Surrogate-assisted evolution, multi-fidelity, hybrids Brief and later: each needs the GP (batch B) and a design settled on real use. From da79a8a0cf88ccb271b67fbe94f8df8ff81ff043 Mon Sep 17 00:00:00 2001 From: tachsin Date: Fri, 2 Oct 2026 12:07:00 +0300 Subject: [PATCH 05/13] feat(model): Gaussian process regression in model::gp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A public Gaussian process, documented as unstable for one release: a constant mean, an ARD Matérn 5/2 kernel by default or the squared exponential, learned or fixed noise, inputs scaled to the unit cube by the bounds of a Real genome and outputs standardized. The hyperparameters maximize the log marginal likelihood (Rasmussen and Williams, eq. 2.30) with the mean at its generalized least squares estimate, by genoxide's own L-BFGS-B on their logarithms with the analytic gradient of eq. 5.9, from a fixed or warm start and random starts on derived streams, the winner by value then start. The posterior mean and variance (eq. 2.25, 2.26, Algorithm 2.1) come with their gradients. Cholesky factorizations by linalg, with a jitter raised tenfold from 1e-10 of the diagonal when needed. L-BFGS-B gains a crate-private minimizer driven by hand with a supplied gradient, for such inner problems. --- src/algorithm/lbfgsb.rs | 40 ++ src/lib.rs | 1 + src/model.rs | 6 + src/model/gp.rs | 1111 +++++++++++++++++++++++++++++++++++++ src/model/gp/tests.rs | 141 +++++ tests/gaussian_process.rs | 305 ++++++++++ 6 files changed, 1604 insertions(+) create mode 100644 src/model.rs create mode 100644 src/model/gp.rs create mode 100644 src/model/gp/tests.rs create mode 100644 tests/gaussian_process.rs diff --git a/src/algorithm/lbfgsb.rs b/src/algorithm/lbfgsb.rs index 416fcc58..1044115e 100644 --- a/src/algorithm/lbfgsb.rs +++ b/src/algorithm/lbfgsb.rs @@ -964,6 +964,46 @@ impl Algorithm for Lbfgsb { } impl Lbfgsb { + /// Minimizes `f` in the box of `real` from `start` with at most `max_evaluations` evaluations, + /// driving the search by hand with the gradient `f` writes: the inner solver of the Gaussian + /// processes' hyperparameters and of Bayesian optimization's acquisition functions. `f` writes + /// the gradient into its second argument (zeroed) and returns the value; a value or gradient + /// that isn't finite is a failed trial, which the line search steps back from. + /// + /// Returns the best point and its value, or `None` if no point had a finite value. No random + /// number is drawn, so the result depends only on `start` and `f`. + pub(crate) fn minimize_with( + real: Real, + start: Reals, + max_evaluations: u64, + mut f: impl FnMut(&[f64], &mut [f64]) -> f64, + ) -> Option<(Reals, f64)> { + let n = start.len(); + let mut lbfgsb = Lbfgsb::builder(real) + .gradients(Gradients::Supplied) + .initial_genome(start) + .minimize() + .seed(0) + .build() + .ok()?; + let mut gradient = vec![0.0; n]; + while lbfgsb.evaluations < max_evaluations && !lbfgsb.is_finished() { + let candidates = lbfgsb.ask(); + let Some(x) = candidates.get(0) else { break }; + gradient.fill(0.0); + let value = f(x, &mut gradient); + let fitness = if value.is_finite() { + Fitness::new(value) + } else { + Fitness::invalid() + }; + lbfgsb.receive(&[fitness], Some(&gradient)).ok()?; + } + let best = lbfgsb.best.take()?; + let value = best.fitness()?.score()?; + value.is_finite().then(|| (best.into_genome(), value)) + } + // the gradients of this run: a stencil for finite differences; an ask under way is asked // again fn set_resolved(&mut self, resolved: Gradients) -> Result<()> { diff --git a/src/lib.rs b/src/lib.rs index 89b688a3..fd7edbac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -108,6 +108,7 @@ pub mod gradient; pub mod individual; pub(crate) mod linalg; pub mod math; +pub mod model; pub mod multi; pub mod neat; pub mod nn; diff --git a/src/model.rs b/src/model.rs new file mode 100644 index 00000000..ea0fecb7 --- /dev/null +++ b/src/model.rs @@ -0,0 +1,6 @@ +//! Surrogate models: cheap approximations of an expensive function, fitted to its evaluations. +//! +//! [`gp`] has Gaussian process regression, the model of +//! [Bayesian optimization](crate::algorithm::bo), which can also be fitted and queried on its own. + +pub mod gp; diff --git a/src/model/gp.rs b/src/model/gp.rs new file mode 100644 index 00000000..375e3e64 --- /dev/null +++ b/src/model/gp.rs @@ -0,0 +1,1111 @@ +//! Gaussian process regression with a constant mean and a stationary kernel, its hyperparameters +//! fitted by maximum marginal likelihood: the surrogate model of +//! [Bayesian optimization](crate::algorithm::bo). +//! +//! **Unstable for one release.** This module is public from genoxide 0.13 so that a Gaussian +//! process can be fitted and queried on its own, but its API and the bits of its fits may still +//! change in 0.14, as the batch and constrained Bayesian optimization of that release use it. +//! +//! A [`GaussianProcess`] models a function `f` of the genes of a [`Real`] genome as +//! `f(x) ~ GP(m, σ_f² k(x, x′))` (Rasmussen and Williams, 2006, eq. 2.37), observed with +//! independent normal noise of variance `σ_n²` (eq. 2.20), and gives the posterior mean and +//! variance of `f` at any point (eq. 2.25 and 2.26), with their gradients. Built with +//! [`GaussianProcess::builder`], fitted with [`fit`](GaussianProcessBuilder::fit): +//! +//! ``` +//! use genoxide::model::gp::GaussianProcess; +//! use genoxide::prelude::*; +//! +//! // a smooth function of one gene, from 8 evaluations +//! let f = |x: f64| x.sin() + 0.1 * x * x; +//! let points: Vec = (0..8).map(|i| Reals::from(vec![f64::from(i)])).collect(); +//! let values: Vec = points.iter().map(|x| f(x[0])).collect(); +//! let gp = GaussianProcess::builder(Real::uniform(1, 0.0..=7.0)?).fit(&points, &values)?; +//! // close to f between the points, and sure of itself there +//! let prediction = gp.predict(&[3.5]); +//! assert!((prediction.mean() - f(3.5)).abs() < 0.05); +//! assert!(prediction.sd() < 0.1); +//! # Ok::<(), genoxide::Error>(()) +//! ``` +//! +//! # The model +//! +//! - **Inputs** are scaled to the unit cube by the bounds of the [`Real`] genome: each gene's +//! range becomes [0, 1], so the length scales of every gene start alike. A gene whose bounds are +//! equal takes no part in the model. +//! - **Outputs** are standardized: the model fits `(y − ȳ) / s`, with `ȳ` and `s` the mean and the +//! standard deviation of the values, and its predictions are mapped back. With given +//! [hyperparameters](GaussianProcessBuilder::hyperparameters), that changes nothing but the +//! rounding: the predictions are those of the model in the values' own units. +//! - **The kernel** ([`Kernel`]) is stationary with a length scale per gene (automatic relevance +//! determination, Rasmussen and Williams, eq. 5.1 and 5.2): `k(x, x′) = k(r)` of the scaled +//! distance `r² = Σᵢ (xᵢ − x′ᵢ)² / ℓᵢ²`. The Matérn kernel with ν = 5/2 by default (eq. 4.17), +//! twice differentiable, as Snoek, Larochelle and Adams (2012) advise for Bayesian optimization +//! against the squared exponential's infinitely smooth functions (eq. 4.9; Stein, 1999). +//! - **The noise** ([`Noise`]) is learned with the other hyperparameters by default, at least +//! 1e-6 of the values' variance; or fixed, e.g. for interpolation. +//! - **The constant mean** `m` is, for given kernel hyperparameters, the one that maximizes the +//! marginal likelihood: the generalized least squares estimate `m = 1ᵀK⁻¹y / 1ᵀK⁻¹1` (setting +//! the derivative of eq. 2.30 with `y − m` for `y` to zero, as eq. 2.38 models a fixed mean). +//! The hyperparameters therefore maximize the likelihood over the mean too, and the mean's +//! derivative being zero there, eq. 5.9 is the gradient of the likelihood so maximized. +//! +//! # Fitting +//! +//! [`fit`](GaussianProcessBuilder::fit) maximizes the log marginal likelihood (eq. 2.30 and 5.8) +//! `ln p(y | X, θ) = −½ (y − m)ᵀ K_y⁻¹ (y − m) − ½ ln |K_y| − (n/2) ln 2π`, `K_y = σ_f² K + σ_n² I`, +//! over the logarithms of the length scales, of `σ_f²` and of `σ_n²`, with genoxide's +//! [`Lbfgsb`](crate::algorithm::Lbfgsb) and the analytic gradient of eq. 5.9, +//! `∂/∂θⱼ ln p = ½ tr((ααᵀ − K_y⁻¹) ∂K_y/∂θⱼ)`, `α = K_y⁻¹(y − m)`. The search runs from +//! [several starts](GaussianProcessBuilder::starts): the first from fixed values (length scales of +//! 0.5 of each range, `σ_f²` the values' variance, `σ_n²` 1e-4 of it, or the noise's least), the +//! others from random points of the box below, drawn from streams derived from the +//! [seed](GaussianProcessBuilder::seed), independent of each other. The likelihood's best wins, +//! the earlier start on ties, so the fit is the same on any number of threads (with the +//! `parallel` feature, the starts run on rayon). The box, in the scaled units: length scales from +//! 0.01 to 100, `σ_f²` from 1e-3 to 1e3, `σ_n²` from the noise's least to 1. +//! +//! Every matrix is factored by Cholesky (Rasmussen and Williams, Algorithm 2.1). A kernel matrix +//! that rounding makes indefinite, e.g. with points very close together and little noise, gets a +//! jitter on its diagonal: none, then 1e-10 of the diagonal's scale, raised tenfold until it +//! factors ([`GaussianProcess::jitter`]). +//! +//! Every operation is a sum, product, quotient or square root in a fixed order, with +//! [`math`](crate::math)'s `exp` and `ln`, so a fit gives the same bits on every platform. +//! +//! References: Rasmussen, C. E. and Williams, C. K. I. (2006). *Gaussian Processes for Machine +//! Learning.* MIT Press, ch. 2, 4 and 5. Snoek, J., Larochelle, H. and Adams, R. P. (2012). +//! Practical Bayesian optimization of machine learning algorithms. *NeurIPS 25*, +//! arXiv:1206.2944. Stein, M. L. (1999). *Interpolation of Spatial Data.* Springer. + +use crate::algorithm::Lbfgsb; +use crate::genome::{Real, Reals, Representation}; +use crate::linalg::cholesky::cholesky_with_jitter; +use crate::linalg::triangular::{ + cholesky_solve, cholesky_solve_multi, solve_lower, solve_lower_transposed, +}; +use crate::math::{exp, ln}; +use crate::{Error, Result, StreamRng}; +use std::ops::RangeInclusive; + +/// The kernel of a [`GaussianProcess`]: the correlation of `f` at two points as a function of +/// their scaled distance `r`, with `r² = Σᵢ (xᵢ − x′ᵢ)² / ℓᵢ²`, a length scale `ℓᵢ` per gene. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +#[non_exhaustive] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Kernel { + /// The Matérn kernel with ν = 5/2, `(1 + √5 r + 5r²/3) exp(−√5 r)` (Rasmussen and Williams, + /// 2006, eq. 4.17): functions twice differentiable. The default, as for most Bayesian + /// optimization (Snoek, Larochelle and Adams, 2012). + #[default] + Matern52, + /// The squared exponential, `exp(−r²/2)` (Rasmussen and Williams, eq. 4.9): infinitely + /// differentiable functions, smoother than most that are optimized. + SquaredExponential, +} + +impl Kernel { + /// `k(r)` and its derivative with respect to `r²`, at `r²`. + #[inline] + fn eval(self, r2: f64) -> (f64, f64) { + match self { + Kernel::Matern52 => { + // s = √5 r: k = (1 + s + s²/3) e^(−s), dk/d(r²) = −(5/6)(1 + s) e^(−s) + let s = (5.0 * r2).sqrt(); + let e = exp(-s); + ((1.0 + s + s * s / 3.0) * e, -5.0 / 6.0 * (1.0 + s) * e) + } + Kernel::SquaredExponential => { + let k = exp(-0.5 * r2); + (k, -0.5 * k) + } + } + } +} + +/// The observation noise of a [`GaussianProcess`]: its variance `σ_n²`, as a fraction of the +/// values' variance (the model's outputs are standardized). +#[derive(Clone, Copy, Debug, PartialEq)] +#[non_exhaustive] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Noise { + /// Learned with the other hyperparameters, at least `min` (0 < `min` < 1). The default is + /// `min` = 1e-6: for a deterministic function, the model then nearly interpolates its values, + /// while the floor keeps the kernel matrix well conditioned. + Learned { + /// The least noise variance, a fraction of the values' variance. + min: f64, + }, + /// A fixed variance, at least 0: 0 interpolates the values (with the jitter the Cholesky + /// factorization may need). + Fixed(f64), +} + +impl Default for Noise { + fn default() -> Self { + Noise::Learned { min: 1e-6 } + } +} + +impl Noise { + pub(crate) fn validate(self) -> Result<()> { + let valid = match self { + Noise::Learned { min } => min > 0.0 && min < 1.0, + Noise::Fixed(variance) => variance >= 0.0 && variance.is_finite(), + }; + if valid { + Ok(()) + } else { + Err(Error::InvalidSetting { + setting: "noise", + reason: format!( + "a learned noise's least variance must be between 0 and 1 (exclusive), and a \ + fixed one finite and at least 0; got {self:?}" + ), + }) + } + } +} + +/// The hyperparameters of a [`GaussianProcess`], in the units of the genes and of the values. +#[derive(Clone, Debug, PartialEq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Hyperparameters { + mean: f64, + length_scales: Vec, + signal_variance: f64, + noise_variance: f64, +} + +impl Hyperparameters { + /// Hyperparameters: the constant mean `m`, a length scale `ℓᵢ` per gene (in the gene's own + /// units; ignored for genes whose bounds are equal), the signal variance `σ_f²` and the noise + /// variance `σ_n²`. + pub fn new( + mean: f64, + length_scales: Vec, + signal_variance: f64, + noise_variance: f64, + ) -> Self { + Self { + mean, + length_scales, + signal_variance, + noise_variance, + } + } + + /// The constant mean `m`. + pub fn mean(&self) -> f64 { + self.mean + } + + /// The length scales `ℓᵢ`, one per gene, in the genes' units: infinite for a gene whose bounds + /// are equal. + pub fn length_scales(&self) -> &[f64] { + &self.length_scales + } + + /// The signal variance `σ_f²`, the variance of `f` far from any data. + pub fn signal_variance(&self) -> f64 { + self.signal_variance + } + + /// The noise variance `σ_n²`. + pub fn noise_variance(&self) -> f64 { + self.noise_variance + } +} + +/// A [`GaussianProcess`]'s prediction at a point: the posterior mean and variance of `f` there +/// (Rasmussen and Williams, 2006, eq. 2.25 and 2.26), without the noise. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Prediction { + mean: f64, + variance: f64, +} + +impl Prediction { + /// The posterior mean. + pub fn mean(&self) -> f64 { + self.mean + } + + /// The posterior variance, at least 0: about 0 at the data (up to the noise), and the signal + /// variance far from it. + pub fn variance(&self) -> f64 { + self.variance + } + + /// The posterior standard deviation, `√variance`. + pub fn sd(&self) -> f64 { + self.variance.sqrt() + } +} + +// the map between genomes and the model's unit cube: the genes with more than one value, each +// range scaled to [0, 1] +#[derive(Clone, Debug)] +pub(crate) struct Scaling { + // the genes' values: the lower bounds, which are the values of the fixed genes + template: Vec, + variable: Vec, + lower: Vec, + upper: Vec, + width: Vec, +} + +impl Scaling { + pub(crate) fn new(real: &Real) -> Self { + let bounds = real.bounds(); + let variable = real.variable_genes().to_vec(); + let lower: Vec = variable.iter().map(|&i| *bounds[i].start()).collect(); + let upper: Vec = variable.iter().map(|&i| *bounds[i].end()).collect(); + let width = lower.iter().zip(&upper).map(|(l, u)| u - l).collect(); + Self { + template: bounds.iter().map(|range| *range.start()).collect(), + variable, + lower, + upper, + width, + } + } + + // the model's dimensions: the genes with more than one value + pub(crate) fn dims(&self) -> usize { + self.variable.len() + } + + pub(crate) fn genes(&self) -> usize { + self.template.len() + } + + // the unit-cube coordinates of a genome + pub(crate) fn to_unit(&self, genome: &[f64], unit: &mut [f64]) { + for (k, &i) in self.variable.iter().enumerate() { + unit[k] = (genome[i] - self.lower[k]) / self.width[k]; + } + } + + // the genome at unit-cube coordinates, in the box: the ends map to the bounds exactly + pub(crate) fn to_genome(&self, unit: &[f64]) -> Reals { + let mut genome = self.template.clone(); + for (k, &i) in self.variable.iter().enumerate() { + let u = unit[k]; + genome[i] = if u >= 1.0 { + self.upper[k] + } else if u <= 0.0 { + self.lower[k] + } else { + (self.lower[k] + u * self.width[k]).clamp(self.lower[k], self.upper[k]) + }; + } + Reals::from(genome) + } + + // a gradient with respect to the unit-cube coordinates, as one with respect to the genes + fn to_gene_gradient(&self, unit_gradient: &[f64], scale: f64, gradient: &mut [f64]) { + gradient.fill(0.0); + for (k, &i) in self.variable.iter().enumerate() { + gradient[i] = scale * unit_gradient[k] / self.width[k]; + } + } +} + +// the settings of a fit +#[derive(Clone, Copy, Debug)] +pub(crate) struct Settings { + pub(crate) kernel: Kernel, + pub(crate) noise: Noise, + pub(crate) starts: usize, + pub(crate) seed: u64, +} + +// the box of the log-hyperparameters, in the scaled units: length scales, σ_f², σ_n² +const LENGTH_SCALES: RangeInclusive = 0.01..=100.0; +const SIGNAL_VARIANCE: RangeInclusive = 1e-3..=1e3; +const MAX_NOISE_VARIANCE: f64 = 1.0; +// the first start's values +const INITIAL_LENGTH_SCALE: f64 = 0.5; +const INITIAL_NOISE_VARIANCE: f64 = 1e-4; +// the evaluations of the likelihood per start +const MAX_EVALUATIONS: u64 = 200; +// the first jitter tried, a fraction of the diagonal's scale +const INITIAL_JITTER: f64 = 1e-10; + +/// A Gaussian process fitted to evaluations of a function: its posterior mean and variance at any +/// point, with their gradients. See the [module](self) for the model and how it's fitted. +/// +/// Built with [`GaussianProcess::builder`]. +#[derive(Clone, Debug)] +pub struct GaussianProcess { + kernel: Kernel, + scaling: Scaling, + // the points in the unit cube, count × dims, and the standardization of the values + x: Vec, + count: usize, + y_mean: f64, + y_scale: f64, + // the hyperparameters in the scaled units, and their logarithms (the fitted parameters) + mean: f64, + length_scales: Vec, + signal: f64, + noise: f64, + log: Vec, + jitter: f64, + // the Cholesky factor L of K_y, α = K_y⁻¹(y − m), and the log marginal likelihood + factor: Vec, + alpha: Vec, + log_likelihood: f64, +} + +impl GaussianProcess { + /// A builder for a Gaussian process on the genes of `real`, whose bounds scale the inputs. + pub fn builder(real: Real) -> GaussianProcessBuilder { + GaussianProcessBuilder { + real, + kernel: Kernel::default(), + noise: Noise::default(), + starts: 5, + seed: 0, + hyperparameters: None, + } + } + + /// The kernel. + pub fn kernel(&self) -> Kernel { + self.kernel + } + + /// The number of points the model was fitted to. + pub fn len(&self) -> usize { + self.count + } + + /// Whether the model has no points: never, as fitting needs at least one. + pub fn is_empty(&self) -> bool { + self.count == 0 + } + + /// The hyperparameters, fitted or given, in the units of the genes and of the values. + pub fn hyperparameters(&self) -> Hyperparameters { + let s2 = self.y_scale * self.y_scale; + let mut length_scales = vec![f64::INFINITY; self.scaling.genes()]; + for (k, &i) in self.scaling.variable.iter().enumerate() { + length_scales[i] = self.length_scales[k] * self.scaling.width[k]; + } + Hyperparameters { + mean: self.y_mean + self.y_scale * self.mean, + length_scales, + signal_variance: s2 * self.signal, + noise_variance: s2 * self.noise, + } + } + + /// The jitter added to the diagonal of the kernel matrix for its Cholesky factorization, in + /// the values' units: usually 0. + pub fn jitter(&self) -> f64 { + self.y_scale * self.y_scale * self.jitter + } + + /// The log marginal likelihood of the values at the hyperparameters (Rasmussen and Williams, + /// 2006, eq. 2.30), in the values' own units. + pub fn log_marginal_likelihood(&self) -> f64 { + self.log_likelihood - self.count as f64 * ln(self.y_scale) + } + + /// The posterior mean and variance of `f` at `genome` (Rasmussen and Williams, 2006, eq. 2.25 + /// and 2.26, as their Algorithm 2.1 computes them). + /// + /// # Panics + /// + /// If `genome` doesn't have a value per gene. + pub fn predict(&self, genome: &[f64]) -> Prediction { + let mut unit = vec![0.0; self.scaling.dims()]; + self.check_len(genome); + self.scaling.to_unit(genome, &mut unit); + let (mean, variance) = self.predict_unit(&unit, None); + self.unscaled(mean, variance) + } + + /// The posterior mean and variance at `genome`, with their gradients with respect to the + /// genes written into `mean_gradient` and `variance_gradient` (0 for a gene whose bounds are + /// equal). Where rounding would make the variance negative, it's 0, with a gradient of 0. + /// + /// # Panics + /// + /// If `genome`, `mean_gradient` or `variance_gradient` doesn't have a value per gene. + pub fn predict_with_gradient( + &self, + genome: &[f64], + mean_gradient: &mut [f64], + variance_gradient: &mut [f64], + ) -> Prediction { + self.check_len(genome); + self.check_len(mean_gradient); + self.check_len(variance_gradient); + let dims = self.scaling.dims(); + let mut unit = vec![0.0; dims]; + self.scaling.to_unit(genome, &mut unit); + let mut dmean = vec![0.0; dims]; + let mut dvariance = vec![0.0; dims]; + let (mean, variance) = self.predict_unit(&unit, Some((&mut dmean, &mut dvariance))); + self.scaling + .to_gene_gradient(&dmean, self.y_scale, mean_gradient); + self.scaling + .to_gene_gradient(&dvariance, self.y_scale * self.y_scale, variance_gradient); + self.unscaled(mean, variance) + } + + fn check_len(&self, values: &[f64]) { + assert_eq!( + values.len(), + self.scaling.genes(), + "a Gaussian process of {} genes", + self.scaling.genes() + ); + } + + fn unscaled(&self, mean: f64, variance: f64) -> Prediction { + Prediction { + mean: self.y_mean + self.y_scale * mean, + variance: self.y_scale * self.y_scale * variance, + } + } + + // the scaled units' mean and variance at the unit-cube point `u`, with their gradients + // with respect to `u` + pub(crate) fn predict_unit( + &self, + u: &[f64], + gradients: Option<(&mut [f64], &mut [f64])>, + ) -> (f64, f64) { + let (n, dims) = (self.count, self.scaling.dims()); + // k(x, xⱼ) and σ_f² dk/d(r²) for each point xⱼ + let mut k = vec![0.0; n]; + let mut dk = vec![0.0; n]; + let mut mean = self.mean; + for j in 0..n { + let xj = &self.x[j * dims..(j + 1) * dims]; + let r2 = scaled_distance(u, xj, &self.length_scales); + let (value, derivative) = self.kernel.eval(r2); + k[j] = self.signal * value; + dk[j] = self.signal * derivative; + mean += self.alpha[j] * k[j]; + } + // v = L⁻¹k, V[f] = σ_f² − vᵀv (Algorithm 2.1, lines 5 and 6) + let mut v = k; + solve_lower(&self.factor, n, &mut v); + let mut vv = 0.0; + for &vj in &v { + vv += vj * vj; + } + let raw_variance = self.signal - vv; + let variance = raw_variance.max(0.0); + if let Some((dmean, dvariance)) = gradients { + // ∂k(x, xⱼ)/∂xᵢ = σ_f² dk/d(r²) · 2 (xᵢ − xⱼᵢ) / ℓᵢ²; ∂V/∂xᵢ = −2 (K_y⁻¹k)ᵀ ∂k/∂xᵢ + let mut w = v; + solve_lower_transposed(&self.factor, n, &mut w); + dmean.fill(0.0); + dvariance.fill(0.0); + for j in 0..n { + let xj = &self.x[j * dims..(j + 1) * dims]; + for i in 0..dims { + let l = self.length_scales[i]; + let dki = dk[j] * 2.0 * (u[i] - xj[i]) / (l * l); + dmean[i] += self.alpha[j] * dki; + dvariance[i] -= 2.0 * w[j] * dki; + } + } + if raw_variance <= 0.0 { + dvariance.fill(0.0); + } + } + (mean, variance) + } + + // the unit-cube coordinates of the model's point `index` + pub(crate) fn unit_point(&self, index: usize) -> &[f64] { + let dims = self.scaling.dims(); + &self.x[index * dims..(index + 1) * dims] + } + + // the logarithms of the fitted hyperparameters, for a warm start of the next fit + pub(crate) fn log_parameters(&self) -> &[f64] { + &self.log + } + + // the standardization of the values: the mean and the scale + pub(crate) fn standardization(&self) -> (f64, f64) { + (self.y_mean, self.y_scale) + } + + // fits the model to the unit-cube points `x` (count × dims) and `values`, all finite, from + // the warm start `warm` (the logarithms of an earlier fit's hyperparameters) or the fixed + // first start + pub(crate) fn fit_unit( + settings: Settings, + scaling: Scaling, + x: Vec, + values: &[f64], + warm: Option<&[f64]>, + ) -> GaussianProcess { + let count = values.len(); + let (y_mean, y_scale, y) = standardize(values); + let dims = scaling.dims(); + let data = Data { + x: &x, + y: &y, + count, + dims, + kernel: settings.kernel, + noise: settings.noise, + }; + let bounds = parameter_bounds(dims, settings.noise); + let first = initial_parameters(dims, settings.noise, &bounds, warm); + let best = maximize_likelihood(&data, &bounds, first.clone(), settings); + let log = best.unwrap_or(first); + let (length_scales, signal, noise) = unpack(&log, dims, settings.noise); + let mut workspace = Workspace::new(count); + // the box's parameters always factor with jitter up to the diagonal's scale, a + // positive definite K_y + (σ_f² + σ_n²) I + let jitter = data + .factor(&length_scales, signal, noise, &mut workspace) + .unwrap_or(f64::NAN); + let (mean, log_likelihood) = data.solve(&mut workspace, None); + let Workspace { l, b, .. } = workspace; + GaussianProcess { + kernel: settings.kernel, + scaling, + x, + count, + y_mean, + y_scale, + mean, + length_scales, + signal, + noise, + log, + jitter, + factor: l, + alpha: b, + log_likelihood, + } + } +} + +// (y − ȳ) / s, with s the standard deviation, 1 if the values are all equal +fn standardize(values: &[f64]) -> (f64, f64, Vec) { + let n = values.len() as f64; + let mut sum = 0.0; + for &y in values { + sum += y; + } + let mean = sum / n; + let mut squares = 0.0; + for &y in values { + squares += (y - mean) * (y - mean); + } + let scale = (squares / n).sqrt(); + let scale = if scale > 0.0 && scale.is_finite() { + scale + } else { + 1.0 + }; + ( + mean, + scale, + values.iter().map(|y| (y - mean) / scale).collect(), + ) +} + +// Σᵢ (aᵢ − bᵢ)² / ℓᵢ² +#[inline] +fn scaled_distance(a: &[f64], b: &[f64], length_scales: &[f64]) -> f64 { + let mut r2 = 0.0; + for i in 0..a.len() { + let t = (a[i] - b[i]) / length_scales[i]; + r2 += t * t; + } + r2 +} + +// the box of the log-hyperparameters: ln ℓᵢ for each gene, ln σ_f², and ln σ_n² when learned +fn parameter_bounds(dims: usize, noise: Noise) -> Vec> { + let log = |range: RangeInclusive| ln(*range.start())..=ln(*range.end()); + let mut bounds = vec![log(LENGTH_SCALES); dims]; + bounds.push(log(SIGNAL_VARIANCE)); + if let Noise::Learned { min } = noise { + bounds.push(log(min..=MAX_NOISE_VARIANCE)); + } + bounds +} + +// the first start: the warm start in the box, or the fixed values +fn initial_parameters( + dims: usize, + noise: Noise, + bounds: &[RangeInclusive], + warm: Option<&[f64]>, +) -> Vec { + if let Some(warm) = warm.filter(|warm| warm.len() == bounds.len()) { + return warm + .iter() + .zip(bounds) + .map(|(&p, range)| p.clamp(*range.start(), *range.end())) + .collect(); + } + let mut start = vec![ln(INITIAL_LENGTH_SCALE); dims]; + start.push(0.0); + if let Noise::Learned { min } = noise { + start.push(ln(INITIAL_NOISE_VARIANCE.max(min))); + } + start +} + +// the length scales, σ_f² and σ_n² of the log-hyperparameters +fn unpack(log: &[f64], dims: usize, noise: Noise) -> (Vec, f64, f64) { + let length_scales = log[..dims].iter().map(|&p| exp(p)).collect(); + let signal = exp(log[dims]); + let noise = match noise { + Noise::Learned { .. } => exp(log[dims + 1]), + Noise::Fixed(variance) => variance, + }; + (length_scales, signal, noise) +} + +// the best of the starts, by L-BFGS-B on the negated log marginal likelihood; None if no start +// gave a finite likelihood +fn maximize_likelihood( + data: &Data<'_>, + bounds: &[RangeInclusive], + first: Vec, + settings: Settings, +) -> Option> { + let Ok(real) = Real::new(bounds.iter().cloned()) else { + return None; + }; + let starts = settings.starts.max(1); + let root = StreamRng::seed_from_u64(settings.seed); + let results = map_in_order(starts, |start| { + let point = if start == 0 { + first.clone() + } else { + real.random_genome(&mut root.derive(start as u64)) + .into_vec() + }; + let mut workspace = Workspace::new(data.count); + Lbfgsb::minimize_with( + real.clone(), + Reals::from(point), + MAX_EVALUATIONS, + |log, gradient| { + let value = data.log_likelihood(log, &mut workspace, Some(gradient)); + for g in gradient.iter_mut() { + *g = -*g; + } + -value + }, + ) + }); + let mut best: Option<(Reals, f64)> = None; + for (point, value) in results.into_iter().flatten() { + if best.as_ref().is_none_or(|(_, best)| value < *best) { + best = Some((point, value)); + } + } + best.map(|(point, _)| point.into_vec()) +} + +// `f(i)` for i in 0..n, in order: on rayon with the `parallel` feature, the same results +pub(crate) fn map_in_order(n: usize, f: impl Fn(usize) -> T + Sync + Send) -> Vec { + #[cfg(feature = "parallel")] + { + use rayon::prelude::*; + (0..n).into_par_iter().map(f).collect() + } + #[cfg(not(feature = "parallel"))] + { + (0..n).map(f).collect() + } +} + +// the matrices of a likelihood's evaluation, reused between evaluations +struct Workspace { + k: Vec, + l: Vec, + inverse: Vec, + a: Vec, + b: Vec, + diff: Vec, +} + +impl Workspace { + fn new(n: usize) -> Self { + Self { + k: vec![0.0; n * n], + l: vec![0.0; n * n], + inverse: Vec::new(), + a: vec![0.0; n], + b: vec![0.0; n], + diff: Vec::new(), + } + } +} + +// the standardized data of a fit +struct Data<'a> { + x: &'a [f64], + y: &'a [f64], + count: usize, + dims: usize, + kernel: Kernel, + noise: Noise, +} + +impl Data<'_> { + // factors K_y = σ_f² K + σ_n² I (plus the jitter it needs) into `workspace.l`: the jitter, + // or None if no jitter up to the diagonal's scale factors it + fn factor( + &self, + length_scales: &[f64], + signal: f64, + noise: f64, + workspace: &mut Workspace, + ) -> Option { + let (n, dims) = (self.count, self.dims); + for p in 0..n { + let xp = &self.x[p * dims..(p + 1) * dims]; + for q in 0..p { + let xq = &self.x[q * dims..(q + 1) * dims]; + let (k, _) = self.kernel.eval(scaled_distance(xp, xq, length_scales)); + workspace.k[p * n + q] = signal * k; + } + workspace.k[p * n + p] = signal + noise; + } + let scale = signal + noise; + if !(scale > 0.0 && scale.is_finite()) { + return None; + } + cholesky_with_jitter( + &workspace.k, + n, + INITIAL_JITTER * scale, + scale, + &mut workspace.l, + ) + .ok() + } + + // from the factor in `workspace.l`: the mean (the generalized least squares estimate, or + // `fixed`), α = K_y⁻¹(y − m) into `workspace.b`, and the log marginal likelihood (eq. 2.30) + fn solve(&self, workspace: &mut Workspace, fixed: Option) -> (f64, f64) { + let n = self.count; + let l = &workspace.l; + // b = K_y⁻¹y and a = K_y⁻¹1 + workspace.b.copy_from_slice(self.y); + cholesky_solve(l, n, &mut workspace.b); + let mean = match fixed { + Some(mean) => { + workspace.a.fill(mean); + cholesky_solve(l, n, &mut workspace.a); + mean + } + None => { + workspace.a.fill(1.0); + cholesky_solve(l, n, &mut workspace.a); + let (mut sa, mut sb) = (0.0, 0.0); + for i in 0..n { + sa += workspace.a[i]; + sb += workspace.b[i]; + } + let mean = sb / sa; + for a in &mut workspace.a { + *a *= mean; + } + mean + } + }; + // α = K_y⁻¹y − K_y⁻¹(m 1) + let mut fit = 0.0; + for i in 0..n { + workspace.b[i] -= workspace.a[i]; + fit += (self.y[i] - mean) * workspace.b[i]; + } + let mut half_log_det = 0.0; + for i in 0..n { + half_log_det += ln(l[i * n + i]); + } + let log_likelihood = -0.5 * fit - half_log_det - 0.5 * n as f64 * ln(std::f64::consts::TAU); + (mean, log_likelihood) + } + + // the log marginal likelihood at the log-hyperparameters `log`, the mean at its best, with + // its gradient (eq. 5.9) into `gradient`; NaN if K_y doesn't factor + fn log_likelihood( + &self, + log: &[f64], + workspace: &mut Workspace, + gradient: Option<&mut [f64]>, + ) -> f64 { + let (n, dims) = (self.count, self.dims); + let (length_scales, signal, noise) = unpack(log, dims, self.noise); + if self + .factor(&length_scales, signal, noise, workspace) + .is_none() + { + return f64::NAN; + } + let (_, value) = self.solve(workspace, None); + let Some(gradient) = gradient else { + return value; + }; + // K_y⁻¹ from its factor, and W = ααᵀ − K_y⁻¹: ∂/∂θⱼ = ½ Σ_pq W_pq ∂K_pq/∂θⱼ, each + // pair p ≠ q twice by symmetry + workspace.inverse.clear(); + workspace.inverse.resize(n * n, 0.0); + for i in 0..n { + workspace.inverse[i * n + i] = 1.0; + } + cholesky_solve_multi(&workspace.l, n, &mut workspace.inverse, n); + workspace.diff.clear(); + workspace.diff.resize(dims, 0.0); + let alpha = &workspace.b; + gradient.fill(0.0); + let mut trace = 0.0; + for p in 0..n { + let xp = &self.x[p * dims..(p + 1) * dims]; + for q in 0..p { + let xq = &self.x[q * dims..(q + 1) * dims]; + let w = alpha[p] * alpha[q] - workspace.inverse[p * n + q]; + let mut r2 = 0.0; + for i in 0..dims { + let t = (xp[i] - xq[i]) / length_scales[i]; + workspace.diff[i] = t * t; + r2 += t * t; + } + let (k, dk) = self.kernel.eval(r2); + // ∂K_pq/∂ln ℓᵢ = σ_f² dk/d(r²) · (−2 uᵢ²), uᵢ = (x_pᵢ − x_qᵢ)/ℓᵢ + let factor = w * signal * dk * -2.0; + for (g, &u2) in gradient.iter_mut().zip(&workspace.diff) { + *g += factor * u2; + } + // ∂K_pq/∂ln σ_f² = σ_f² k_pq + gradient[dims] += w * signal * k; + } + trace += alpha[p] * alpha[p] - workspace.inverse[p * n + p]; + } + // the diagonal: ∂K_pp/∂ln σ_f² = σ_f², ∂K_pp/∂ln σ_n² = σ_n² + gradient[dims] += 0.5 * signal * trace; + if let Noise::Learned { .. } = self.noise { + gradient[dims + 1] = 0.5 * noise * trace; + } + value + } +} + +/// A builder for a [`GaussianProcess`], from [`GaussianProcess::builder`]. +/// +/// Defaults: the [Matérn 5/2 kernel](Kernel::Matern52), [learned noise](Noise::Learned) of at +/// least 1e-6 of the values' variance, 5 starts of the likelihood's maximization, seed 0. +#[derive(Clone, Debug)] +pub struct GaussianProcessBuilder { + real: Real, + kernel: Kernel, + noise: Noise, + starts: usize, + seed: u64, + hyperparameters: Option, +} + +impl GaussianProcessBuilder { + /// The kernel: [`Kernel::Matern52`] by default. + pub fn kernel(mut self, kernel: Kernel) -> Self { + self.kernel = kernel; + self + } + + /// The observation noise: [`Noise::Learned`] with a least variance of 1e-6 of the values' + /// variance by default. + pub fn noise(mut self, noise: Noise) -> Self { + self.noise = noise; + self + } + + /// The number of starts of the likelihood's maximization, at least 1: 5 by default. The + /// first from fixed values, the others from random points. + pub fn starts(mut self, starts: usize) -> Self { + self.starts = starts; + self + } + + /// The seed of the random starts: 0 by default, so a fit is reproducible unless told + /// otherwise. + pub fn seed(mut self, seed: u64) -> Self { + self.seed = seed; + self + } + + /// Hyperparameters to use as they are, instead of fitting them: the model is then the + /// posterior of Rasmussen and Williams (2006, eq. 2.38, 2.25 and 2.26) with this mean and + /// these variances and length scales. The [noise](GaussianProcessBuilder::noise) setting is + /// ignored. + pub fn hyperparameters(mut self, hyperparameters: Hyperparameters) -> Self { + self.hyperparameters = Some(hyperparameters); + self + } + + /// Fits the model to the `values` of a function at `points`. + /// + /// # Errors + /// + /// - [`Error::InvalidSetting`] for no points, a different number of values, a point without + /// a value per gene, a point or value that isn't finite, starts of 0, an invalid + /// [`Noise`], or given hyperparameters without a length scale per gene, with a length scale + /// or signal variance that isn't positive and finite, a negative or infinite noise + /// variance, or a mean that isn't finite. + pub fn fit(&self, points: &[Reals], values: &[f64]) -> Result { + let invalid = + |setting: &'static str, reason: String| Err(Error::InvalidSetting { setting, reason }); + if points.is_empty() { + return invalid("points", "a Gaussian process needs at least 1 point".into()); + } + if values.len() != points.len() { + return invalid( + "values", + format!( + "expected a value per point, {}, got {}", + points.len(), + values.len() + ), + ); + } + let genes = self.real.genome_len(); + for (index, point) in points.iter().enumerate() { + if point.len() != genes || !point.iter().all(|x| x.is_finite()) { + return invalid( + "points", + format!("point {index} must have {genes} finite genes, got {point:?}"), + ); + } + } + if let Some(index) = values.iter().position(|y| !y.is_finite()) { + return invalid( + "values", + format!("value {index} isn't finite: {}", values[index]), + ); + } + if self.starts == 0 { + return invalid("starts", "must be at least 1, got 0".into()); + } + self.noise.validate()?; + let scaling = Scaling::new(&self.real); + let dims = scaling.dims(); + let mut x = vec![0.0; points.len() * dims]; + for (point, unit) in points.iter().zip(x.chunks_exact_mut(dims.max(1))) { + scaling.to_unit(point, unit); + } + if dims == 0 { + x.clear(); + } + let settings = Settings { + kernel: self.kernel, + noise: self.noise, + starts: self.starts, + seed: self.seed, + }; + match &self.hyperparameters { + None => Ok(GaussianProcess::fit_unit( + settings, scaling, x, values, None, + )), + Some(h) => self.with_hyperparameters(h, settings, scaling, x, values), + } + } + + // the model with given hyperparameters, in the scaled units + fn with_hyperparameters( + &self, + h: &Hyperparameters, + settings: Settings, + scaling: Scaling, + x: Vec, + values: &[f64], + ) -> Result { + let invalid = |reason: String| { + Err(Error::InvalidSetting { + setting: "hyperparameters", + reason, + }) + }; + if h.length_scales.len() != scaling.genes() { + return invalid(format!( + "expected a length scale per gene, {}, got {}", + scaling.genes(), + h.length_scales.len() + )); + } + let positive = |x: f64| x > 0.0 && x.is_finite(); + if !scaling + .variable + .iter() + .all(|&i| positive(h.length_scales[i])) + || !positive(h.signal_variance) + || !(h.noise_variance >= 0.0 && h.noise_variance.is_finite()) + || !h.mean.is_finite() + { + return invalid(format!( + "length scales and the signal variance must be positive and finite, the noise \ + variance finite and at least 0, and the mean finite; got {h:?}" + )); + } + let count = values.len(); + let (y_mean, y_scale, y) = standardize(values); + let s2 = y_scale * y_scale; + let length_scales: Vec = scaling + .variable + .iter() + .enumerate() + .map(|(k, &i)| h.length_scales[i] / scaling.width[k]) + .collect(); + let (signal, noise) = (h.signal_variance / s2, h.noise_variance / s2); + let mean = (h.mean - y_mean) / y_scale; + let data = Data { + x: &x, + y: &y, + count, + dims: scaling.dims(), + kernel: settings.kernel, + noise: Noise::Fixed(noise), + }; + let mut workspace = Workspace::new(count); + let Some(jitter) = data.factor(&length_scales, signal, noise, &mut workspace) else { + return invalid(format!( + "the kernel matrix doesn't factor with these hyperparameters: {h:?}" + )); + }; + let (mean, log_likelihood) = data.solve(&mut workspace, Some(mean)); + let mut log: Vec = length_scales.iter().map(|&l| ln(l)).collect(); + log.push(ln(signal)); + let Workspace { l, b, .. } = workspace; + Ok(GaussianProcess { + kernel: settings.kernel, + scaling, + x, + count, + y_mean, + y_scale, + mean, + length_scales, + signal, + noise, + log, + jitter, + factor: l, + alpha: b, + log_likelihood, + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/src/model/gp/tests.rs b/src/model/gp/tests.rs new file mode 100644 index 00000000..dbbe3191 --- /dev/null +++ b/src/model/gp/tests.rs @@ -0,0 +1,141 @@ +use super::*; + +// a fixed pseudo-random sequence in [0, 1), so the tests don't depend on a generator +fn sequence(n: usize, seed: u64) -> Vec { + let mut state = seed; + (0..n) + .map(|_| { + state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1_442_695_040_888_963_407); + (state >> 11) as f64 / (1u64 << 53) as f64 + }) + .collect() +} + +#[test] +fn kernels_are_their_formulas() { + for r in [0.0, 1e-8, 0.3, 1.0, 2.5, 7.0] { + let r2 = r * r; + // Rasmussen and Williams, eq. 4.17 and 4.9 + let sqrt5 = 5.0_f64.sqrt(); + let matern = (1.0 + sqrt5 * r + 5.0 * r * r / 3.0) * (-sqrt5 * r).exp(); + let (k, _) = Kernel::Matern52.eval(r2); + assert!( + (k - matern).abs() <= 1e-15, + "Matérn at r = {r}: {k} against {matern}" + ); + let (k, _) = Kernel::SquaredExponential.eval(r2); + assert!((k - (-0.5 * r2).exp()).abs() <= 1e-15); + } + // the derivatives with respect to r², against central differences + for kernel in [Kernel::Matern52, Kernel::SquaredExponential] { + for r2 in [0.01, 0.4, 1.0, 3.0] { + let h = 1e-6; + let numeric = (kernel.eval(r2 + h).0 - kernel.eval(r2 - h).0) / (2.0 * h); + let (_, analytic) = kernel.eval(r2); + assert!( + (numeric - analytic).abs() <= 1e-8, + "{kernel:?} at r² = {r2}: {analytic} against {numeric}" + ); + } + // at 0: the correlation 1, and a finite slope + assert_eq!(kernel.eval(0.0).0, 1.0); + assert!(kernel.eval(0.0).1.is_finite()); + } +} + +// standardized values of a smooth function at `count` points of `dims` genes +fn data(count: usize, dims: usize, seed: u64) -> (Vec, Vec) { + let x = sequence(count * dims, seed); + let values: Vec = x + .chunks_exact(dims) + .map(|p| { + p.iter() + .enumerate() + .map(|(i, &v)| (3.0 * v + i as f64).sin()) + .sum() + }) + .collect(); + let (_, _, y) = standardize(&values); + (x, y) +} + +#[test] +fn log_likelihood_gradient_matches_central_differences() { + for kernel in [Kernel::Matern52, Kernel::SquaredExponential] { + for noise in [Noise::Learned { min: 1e-6 }, Noise::Fixed(1e-3)] { + let (count, dims) = (9, 3); + let (x, y) = data(count, dims, 7); + let data = Data { + x: &x, + y: &y, + count, + dims, + kernel, + noise, + }; + let mut workspace = Workspace::new(count); + let points = [ + vec![-0.7, 0.2, -1.2, 0.3, -5.0], + vec![0.5, -0.4, 0.1, -0.8, -2.0], + vec![-1.5, -1.0, 0.0, 1.0, -9.0], + ]; + for (case, point) in points.into_iter().enumerate() { + let parameters = match noise { + Noise::Learned { .. } => point, + Noise::Fixed(_) => point[..dims + 1].to_vec(), + }; + let mut gradient = vec![0.0; parameters.len()]; + let value = data.log_likelihood(¶meters, &mut workspace, Some(&mut gradient)); + assert!(value.is_finite()); + for j in 0..parameters.len() { + let h = 1e-5; + let mut plus = parameters.clone(); + plus[j] += h; + let mut minus = parameters.clone(); + minus[j] -= h; + let numeric = (data.log_likelihood(&plus, &mut workspace, None) + - data.log_likelihood(&minus, &mut workspace, None)) + / (2.0 * h); + let error = (numeric - gradient[j]).abs(); + assert!( + error <= 1e-6 * gradient[j].abs().max(1.0), + "{kernel:?}, {noise:?}, case {case}, parameter {j}: {} against {numeric}", + gradient[j] + ); + } + } + } + } +} + +#[test] +fn the_mean_maximizes_the_likelihood() { + // the generalized least squares mean: the likelihood with any other mean is lower + let (count, dims) = (8, 2); + let (x, y) = data(count, dims, 3); + let data = Data { + x: &x, + y: &y, + count, + dims, + kernel: Kernel::Matern52, + noise: Noise::Fixed(1e-4), + }; + let mut workspace = Workspace::new(count); + let (length_scales, signal, noise) = unpack(&[-0.5, 0.1, 0.3], dims, data.noise); + data.factor(&length_scales, signal, noise, &mut workspace) + .unwrap(); + let (mean, best) = data.solve(&mut workspace, None); + for offset in [-0.1, -1e-3, 1e-3, 0.1] { + let (_, other) = data.solve(&mut workspace, Some(mean + offset)); + assert!( + other < best, + "the mean {mean} + {offset}: {other} against {best}" + ); + } + // the same likelihood as the fixed mean at the estimate + let (_, fixed) = data.solve(&mut workspace, Some(mean)); + assert!((fixed - best).abs() <= 1e-12 * best.abs()); +} diff --git a/tests/gaussian_process.rs b/tests/gaussian_process.rs new file mode 100644 index 00000000..6fe52a6f --- /dev/null +++ b/tests/gaussian_process.rs @@ -0,0 +1,305 @@ +//! The Gaussian process of `model::gp`: Rasmussen and Williams's (2006) formulas on cases small +//! enough to work out by hand, interpolation, the posterior's gradients and the fit. + +use genoxide::model::gp::{GaussianProcess, Hyperparameters, Kernel, Noise}; +use genoxide::prelude::*; + +// the kernels' correlation at distance r (Rasmussen and Williams, eq. 4.17 and 4.9) +fn correlation(kernel: Kernel, r: f64) -> f64 { + match kernel { + Kernel::Matern52 => { + let s = 5.0_f64.sqrt() * r; + (1.0 + s + s * s / 3.0) * (-s).exp() + } + _ => (-0.5 * r * r).exp(), + } +} + +// the posterior of two points worked out by hand: K_y = σ_f² [[1, c], [c, 1]] + σ_n² I, its +// inverse [[a, −b], [−b, a]] / det; the mean m + k*ᵀ K_y⁻¹ (y − m) (eq. 2.38), the variance +// σ_f² − k*ᵀ K_y⁻¹ k* (eq. 2.26), and the log marginal likelihood (eq. 2.30) +struct TwoPoints { + mean: f64, + variance: f64, + log_likelihood: f64, +} + +#[allow(clippy::too_many_arguments)] +fn two_points( + kernel: Kernel, + x: [f64; 2], + y: [f64; 2], + at: f64, + length: f64, + signal: f64, + noise: f64, + mean: f64, +) -> TwoPoints { + let c = correlation(kernel, (x[0] - x[1]).abs() / length); + let (a, b) = (signal + noise, signal * c); + let det = a * a - b * b; + let k = [ + signal * correlation(kernel, (at - x[0]).abs() / length), + signal * correlation(kernel, (at - x[1]).abs() / length), + ]; + let r = [y[0] - mean, y[1] - mean]; + let alpha = [(a * r[0] - b * r[1]) / det, (a * r[1] - b * r[0]) / det]; + let kinv_k = [(a * k[0] - b * k[1]) / det, (a * k[1] - b * k[0]) / det]; + TwoPoints { + mean: mean + k[0] * alpha[0] + k[1] * alpha[1], + variance: signal - (k[0] * kinv_k[0] + k[1] * kinv_k[1]), + log_likelihood: -0.5 * (r[0] * alpha[0] + r[1] * alpha[1]) + - 0.5 * det.ln() + - (2.0 * std::f64::consts::PI).ln(), + } +} + +fn close(a: f64, b: f64, tolerance: f64) -> bool { + (a - b).abs() <= tolerance * b.abs().max(1.0) +} + +#[test] +fn two_points_are_the_formulas_worked_out_by_hand() { + for kernel in [Kernel::Matern52, Kernel::SquaredExponential] { + // a gene in [−2, 6]: the length scale 2 is 0.25 of the range in the model's unit cube + let real = Real::uniform(1, -2.0..=6.0).unwrap(); + let (x, y) = ([0.5, 2.0], [1.0, 4.0]); + let (length, signal, noise, mean) = (2.0, 3.0, 0.1, 1.5); + let points: Vec = x.iter().map(|&x| Reals::from(vec![x])).collect(); + let gp = GaussianProcess::builder(real) + .kernel(kernel) + .hyperparameters(Hyperparameters::new(mean, vec![length], signal, noise)) + .fit(&points, &y) + .unwrap(); + for at in [-2.0, 0.5, 1.0, 1.7, 3.0, 6.0] { + let expected = two_points(kernel, x, y, at, length, signal, noise, mean); + let prediction = gp.predict(&[at]); + assert!( + close(prediction.mean(), expected.mean, 1e-13), + "{kernel:?} at {at}: mean {} against {}", + prediction.mean(), + expected.mean + ); + assert!( + close(prediction.variance(), expected.variance, 1e-12), + "{kernel:?} at {at}: variance {} against {}", + prediction.variance(), + expected.variance + ); + assert!(close( + gp.log_marginal_likelihood(), + expected.log_likelihood, + 1e-13 + )); + } + // the hyperparameters come back in the genes' and values' units + let h = gp.hyperparameters(); + assert!(close(h.mean(), mean, 1e-15)); + assert!(close(h.length_scales()[0], length, 1e-15)); + assert!(close(h.signal_variance(), signal, 1e-15)); + assert!(close(h.noise_variance(), noise, 1e-15)); + assert_eq!(gp.jitter(), 0.0); + } +} + +// a smooth function of three genes, the third fixed +fn smooth(x: &[f64]) -> f64 { + (2.0 * x[0]).sin() + 0.5 * x[1] * x[1] - 0.3 * x[0] * x[1] + x[2] +} + +fn sample(count: usize) -> (Real, Vec, Vec) { + let real = Real::new([-1.0..=2.0, 0.0..=3.0, 0.5..=0.5]).unwrap(); + let points = real + .latin_hypercube(count, &mut StreamRng::seed_from_u64(4)) + .unwrap(); + let values = points.iter().map(|x| smooth(x)).collect(); + (real, points, values) +} + +#[test] +fn without_noise_the_mean_interpolates_and_the_variance_vanishes_at_the_data() { + let (real, points, values) = sample(12); + let gp = GaussianProcess::builder(real) + .noise(Noise::Fixed(0.0)) + .fit(&points, &values) + .unwrap(); + for (point, &value) in points.iter().zip(&values) { + let prediction = gp.predict(point); + assert!( + (prediction.mean() - value).abs() <= 1e-6, + "{} against {value}", + prediction.mean() + ); + assert!(prediction.variance() <= 1e-8 * gp.hyperparameters().signal_variance()); + } + // between the points, a good model of a smooth function + let prediction = gp.predict(&[0.4, 1.2, 0.5]); + assert!((prediction.mean() - smooth(&[0.4, 1.2, 0.5])).abs() < 0.05); + assert!(prediction.variance() > 0.0); +} + +#[test] +fn far_from_the_data_the_variance_is_the_signal_variance() { + let real = Real::uniform(1, 0.0..=100.0).unwrap(); + let points: Vec = [1.0, 2.0, 3.0].map(|x| Reals::from(vec![x])).to_vec(); + let gp = GaussianProcess::builder(real) + .hyperparameters(Hyperparameters::new(0.0, vec![0.5], 2.0, 1e-6)) + .fit(&points, &[0.3, -0.2, 0.5]) + .unwrap(); + let far = gp.predict(&[90.0]); + assert!(close(far.variance(), 2.0, 1e-12)); + assert!(close(far.mean(), 0.0, 1e-12)); +} + +#[test] +fn the_posterior_gradients_match_central_differences() { + let (real, points, values) = sample(15); + for kernel in [Kernel::Matern52, Kernel::SquaredExponential] { + let gp = GaussianProcess::builder(real.clone()) + .kernel(kernel) + .fit(&points, &values) + .unwrap(); + for at in [[0.1, 0.4, 0.5], [1.7, 2.9, 0.5], [-0.8, 1.5, 0.5]] { + let (mut dmean, mut dvariance) = ([0.0; 3], [0.0; 3]); + let prediction = gp.predict_with_gradient(&at, &mut dmean, &mut dvariance); + assert_eq!(prediction, gp.predict(&at)); + // the fixed gene takes no part + assert_eq!((dmean[2], dvariance[2]), (0.0, 0.0)); + for i in 0..2 { + let h = 1e-6; + let (mut plus, mut minus) = (at, at); + plus[i] += h; + minus[i] -= h; + let (p, m) = (gp.predict(&plus), gp.predict(&minus)); + let numeric_mean = (p.mean() - m.mean()) / (2.0 * h); + let numeric_variance = (p.variance() - m.variance()) / (2.0 * h); + assert!( + close(dmean[i], numeric_mean, 1e-6), + "{kernel:?} at {at:?}, gene {i}: {} against {numeric_mean}", + dmean[i] + ); + assert!( + (dvariance[i] - numeric_variance).abs() <= 1e-6, + "{kernel:?} at {at:?}, gene {i}: {} against {numeric_variance}", + dvariance[i] + ); + } + } + } +} + +#[test] +fn the_fit_maximizes_the_likelihood_and_is_reproducible() { + let (real, points, values) = sample(20); + let fit = |seed| { + GaussianProcess::builder(real.clone()) + .seed(seed) + .fit(&points, &values) + .unwrap() + }; + let gp = fit(0); + let again = fit(0); + assert_eq!(gp.hyperparameters(), again.hyperparameters()); + assert_eq!( + gp.log_marginal_likelihood().to_bits(), + again.log_marginal_likelihood().to_bits() + ); + // better than the fixed first start, and than nearby hyperparameters + let h = gp.hyperparameters(); + let at = |length_factor: f64, signal_factor: f64| { + let lengths: Vec = h + .length_scales() + .iter() + .map(|l| l * length_factor) + .collect(); + GaussianProcess::builder(real.clone()) + .hyperparameters(Hyperparameters::new( + h.mean(), + lengths, + h.signal_variance() * signal_factor, + h.noise_variance(), + )) + .fit(&points, &values) + .unwrap() + .log_marginal_likelihood() + }; + let best = gp.log_marginal_likelihood(); + assert!(close(at(1.0, 1.0), best, 1e-9)); + for (length, signal) in [(1.1, 1.0), (0.9, 1.0), (1.0, 1.2), (1.0, 0.8)] { + assert!(at(length, signal) < best); + } +} + +#[test] +fn repeated_points_factor_with_jitter() { + let real = Real::uniform(2, 0.0..=1.0).unwrap(); + let points: Vec = [[0.2, 0.3], [0.2, 0.3], [0.7, 0.1], [0.4, 0.9]] + .map(|x| Reals::from(x.to_vec())) + .to_vec(); + let values = [1.0, 1.0, 0.0, 2.0]; + // without noise, a singular kernel matrix: it factors with a jitter + let gp = GaussianProcess::builder(real.clone()) + .hyperparameters(Hyperparameters::new(1.0, vec![0.5, 0.5], 1.0, 0.0)) + .fit(&points, &values) + .unwrap(); + assert!(gp.jitter() > 0.0 && gp.jitter() < 1e-6, "{}", gp.jitter()); + assert!((gp.predict(&[0.2, 0.3]).mean() - 1.0).abs() < 1e-6); + // fitted, with or without noise + for noise in [Noise::Fixed(0.0), Noise::default()] { + let gp = GaussianProcess::builder(real.clone()) + .noise(noise) + .fit(&points, &values) + .unwrap(); + assert!((gp.predict(&[0.2, 0.3]).mean() - 1.0).abs() < 1e-3); + } +} + +#[test] +fn invalid_settings_are_errors() { + let real = Real::uniform(2, 0.0..=1.0).unwrap(); + let points = vec![Reals::from(vec![0.1, 0.2]), Reals::from(vec![0.5, 0.5])]; + let builder = GaussianProcess::builder(real); + let setting = |result: Result| match result { + Err(Error::InvalidSetting { setting, .. }) => setting, + other => panic!("not an invalid setting: {other:?}"), + }; + assert_eq!(setting(builder.fit(&[], &[])), "points"); + assert_eq!(setting(builder.fit(&points, &[1.0])), "values"); + assert_eq!(setting(builder.fit(&points, &[1.0, f64::NAN])), "values"); + let short = vec![Reals::from(vec![0.1])]; + assert_eq!(setting(builder.fit(&short, &[1.0])), "points"); + let infinite = vec![Reals::from(vec![0.1, f64::INFINITY])]; + assert_eq!(setting(builder.fit(&infinite, &[1.0])), "points"); + assert_eq!( + setting(builder.clone().starts(0).fit(&points, &[1.0, 2.0])), + "starts" + ); + for noise in [ + Noise::Learned { min: 0.0 }, + Noise::Learned { min: 1.0 }, + Noise::Fixed(-1.0), + Noise::Fixed(f64::NAN), + ] { + let result = builder.clone().noise(noise).fit(&points, &[1.0, 2.0]); + assert_eq!(setting(result), "noise"); + } + for h in [ + Hyperparameters::new(0.0, vec![1.0], 1.0, 0.0), + Hyperparameters::new(0.0, vec![1.0, 0.0], 1.0, 0.0), + Hyperparameters::new(0.0, vec![1.0, 1.0], -1.0, 0.0), + Hyperparameters::new(0.0, vec![1.0, 1.0], 1.0, -1.0), + Hyperparameters::new(f64::NAN, vec![1.0, 1.0], 1.0, 0.0), + ] { + let result = builder.clone().hyperparameters(h).fit(&points, &[1.0, 2.0]); + assert_eq!(setting(result), "hyperparameters"); + } +} + +#[test] +#[should_panic(expected = "a Gaussian process of 2 genes")] +fn a_point_of_the_wrong_length_panics() { + let real = Real::uniform(2, 0.0..=1.0).unwrap(); + let points = vec![Reals::from(vec![0.1, 0.2])]; + let gp = GaussianProcess::builder(real).fit(&points, &[1.0]).unwrap(); + let _ = gp.predict(&[0.5]); +} From 6cda04fdf549fd86916fb02d3a647083d0f8b086 Mon Sep 17 00:00:00 2001 From: tachsin Date: Fri, 2 Oct 2026 12:07:08 +0300 Subject: [PATCH 06/13] feat(bo): Bayesian optimization, Bo An ask / tell algorithm on Real genomes for expensive black-box functions: an initial design of 2(n + 1) points by default (the initial genomes, then a Latin hypercube), then one point per generation, chosen by fitting the Gaussian process of model::gp to every evaluation and maximizing an acquisition function from raw samples and the best point evaluated with L-BFGS-B and the acquisition's analytic gradient. Log-EI by default, EI, PI (maximized through its logarithm) and UCB, settable during a run. An output transform: the values standardized by default, or the logarithm of their distance above the best for objectives that span orders of magnitude. Invalid points enter the model at the worst value; a point is never asked twice. Reproducible on any platform and thread count, re-evaluation and checkpoints included. The acquisition functions gain their derivatives with respect to the mean and the standard deviation, log-EI's through an asymptotic series where the Mills ratio's difference cancels. --- src/algorithm.rs | 1 + src/algorithm/bo.rs | 1014 ++++++++++++++++++++++++++++++- src/algorithm/bo/acquisition.rs | 159 +++++ src/prelude.rs | 1 + tests/algorithms.rs | 27 +- tests/bo.rs | 386 ++++++++++++ tests/checkpoint.rs | 58 ++ 7 files changed, 1643 insertions(+), 3 deletions(-) create mode 100644 tests/bo.rs diff --git a/src/algorithm.rs b/src/algorithm.rs index 7d5b11fa..365384e2 100644 --- a/src/algorithm.rs +++ b/src/algorithm.rs @@ -45,6 +45,7 @@ pub mod open_es; pub mod pso; pub mod steady; +pub use bo::{Bo, BoBuilder}; pub use cmaes::{Cmaes, CmaesBuilder, Covariance, Restarts}; pub use continuation::{Continuation, ContinuationBuilder, Continue, Keep}; pub use de::{De, DeBuilder}; diff --git a/src/algorithm/bo.rs b/src/algorithm/bo.rs index 3b48c515..e22b73f9 100644 --- a/src/algorithm/bo.rs +++ b/src/algorithm/bo.rs @@ -1,7 +1,1017 @@ //! Bayesian optimization: a surrogate model of an expensive function, and acquisition functions //! that pick where to evaluate it next. //! -//! [`acquisition`] has the acquisition functions, from a model's predictive mean and standard -//! deviation at a point: they work with any surrogate model that gives those. +//! See [`Bo`]. [`acquisition`] has the acquisition functions, from a model's predictive mean and +//! standard deviation at a point: they work with any surrogate model that gives those. pub mod acquisition; + +use super::{Algorithm, Candidates, Lbfgsb, Reevaluate}; +use crate::genome::{Real, Reals, Representation}; +use crate::model::gp::{self, GaussianProcess, Kernel, Noise, Scaling, map_in_order}; +use crate::{Error, Fitness, Individual, Objective, Population, Result, StreamRng}; +use rand::Rng; + +/// The acquisition function of a [`Bo`]: how much a point is worth evaluating, from the model's +/// posterior mean `μ` and standard deviation `σ` there. See [`acquisition`] for the formulas. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +#[non_exhaustive] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Acquisition { + /// The expected improvement over the best value (Močkus, 1975; Jones, Schonlau and Welch, + /// 1998). It underflows to 0 with its gradient far from the best, so its maximization from + /// most starts goes nowhere once the model is sure of itself; + /// [`LogExpectedImprovement`](Acquisition::LogExpectedImprovement) doesn't. + ExpectedImprovement, + /// The logarithm of the expected improvement, computed so that it and its gradient stay + /// finite where the expected improvement underflows (Ament, Daulton, Eriksson, Balandat and + /// Bakshy, 2023): the same maximizer, found far more reliably. The default. + #[default] + LogExpectedImprovement, + /// The probability of improving on the best by more than `xi` (Kushner, 1964), maximized + /// through its logarithm, which has the same maximizer and keeps a gradient where the + /// probability underflows. + ProbabilityOfImprovement { + /// ξ ≥ 0, in the units the model fits ([`Output`]): larger asks for larger improvements, + /// exploring more. 0 is the pure probability of improvement, which exploits greedily. + xi: f64, + }, + /// The confidence bound `μ − √β σ`, minimized (`μ + √β σ` maximized when maximizing): + /// Srinivas, Krause, Kakade and Seeger (2010). Changeable during a run for a schedule + /// ([`Bo::set_acquisition`]). + UpperConfidenceBound { + /// β ≥ 0: 0 exploits the model's mean alone, larger explores more. + beta: f64, + }, +} + +impl Acquisition { + fn validate(self) -> Result<()> { + let parameter = match self { + Acquisition::ProbabilityOfImprovement { xi } => Some(("xi", xi)), + Acquisition::UpperConfidenceBound { beta } => Some(("beta", beta)), + _ => None, + }; + match parameter { + Some((name, value)) if !(value >= 0.0 && value.is_finite()) => { + Err(Error::InvalidSetting { + setting: "acquisition", + reason: format!("{name} must be finite and at least 0, got {value}"), + }) + } + _ => Ok(()), + } + } +} + +/// What the model of a [`Bo`] fits: a transform of the values to minimize (the scores, negated +/// when maximizing), which the model then standardizes. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +#[non_exhaustive] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Output { + /// The values themselves (the default), standardized as Gaussian processes usually are. + #[default] + Standardize, + /// The logarithm of each value's distance above the best, `ln(v − v_best + δ)`, with `δ` the + /// first quartile of the distances (the ⌊N/4⌋-th smallest of N, the best's own 0 counted): + /// for objectives that span orders of magnitude, such as Goldstein-Price's from 3 to 10⁶, + /// whose large values would otherwise flatten the model where the best ones are. A monotone + /// transform: the best value stays the best, and the model resolves small differences near it + /// and compresses large ones far from it. `δ` grows with the spread of the values near the + /// best, so the best point is never an outlier far below the others (as it would be with a + /// tiny `δ`), and shrinks as the search closes in. + /// + /// Measured on 20 seeds per problem (default settings otherwise, to f* + 1e-3): Goldstein-Price + /// in [−2, 2]² reached in 20 runs of 20 within 80 evaluations (0 with + /// [`Standardize`](Output::Standardize)), the six-hump camel in [−5, 5]² in 20 (6), Branin in + /// a median of 25 evaluations (30). + Log, +} + +// the ids of the random streams derived from the seed: they decide the results of seeded runs and +// must never change for the same major version +mod streams { + // the Latin hypercube of the initial design + pub(super) const DESIGN: u64 = 0; + // the starts of the hyperparameters' maximization, per generation + pub(super) const HYPERPARAMETERS: u64 = 1; + // the raw samples of the acquisition's maximization, per generation + pub(super) const RAW_SAMPLES: u64 = 2; + // a random point when there's no model to ask, per generation + pub(super) const RANDOM: u64 = 3; +} + +// the evaluations of the acquisition function per start of its maximization +const ACQUISITION_EVALUATIONS: u64 = 200; +// the least variance of the model's prediction, in its standardized units, for the acquisition +const VARIANCE_FLOOR: f64 = 1e-12; + +/// Bayesian optimization on [`Real`] genomes, as an ask / tell [`Algorithm`]: for expensive +/// black-box functions, such as a simulation that runs for minutes or a physical experiment, where +/// tens to a few hundred evaluations must do. A Gaussian process ([`model::gp`](crate::model::gp)) +/// models the function from every evaluation so far, and an [`Acquisition`] function of its +/// posterior picks the next point to evaluate, trading the model's best guesses against its +/// uncertainty. +/// +/// - **The initial design** (generation 0): [`initial_points`](BoBuilder::initial_points) points, +/// 2(n + 1) for n searched genes by default, the +/// [`initial_genomes`](BoBuilder::initial_genomes) first and a Latin hypercube sample (McKay, +/// Beckman and Conover, 1979; [`Real::latin_hypercube`]) for the rest. A small design leaves +/// most of the budget to the model's choices. For an accurate model of the whole box rather +/// than its minimum, Loeppky, Sacks and Welch (2009) recommend 10n points. +/// - **Each later generation** asks one point: the model is fitted to every evaluation (its +/// hyperparameters by maximum likelihood, from the last fit's and random starts), then the +/// acquisition function is maximized in the box: evaluated at +/// [`raw_samples`](BoBuilder::raw_samples) random points, then improved by [`Lbfgsb`] with its +/// analytic gradient from the best [`acquisition_starts`](BoBuilder::acquisition_starts) of them +/// and from the best point evaluated so far. The best result that isn't an evaluated point is +/// asked: a point is never asked twice. +/// - **The model** fits the values to minimize, the scores negated when maximizing, through the +/// [`Output`] transform. A point with an invalid fitness, or a score that isn't finite, enters +/// the model at the worst value of the others, so the search learns to avoid where the function +/// fails; until a point has a valid score, the next point is random. The search uses the score +/// only: a constraint violation is ignored by it (use a penalty), though +/// [`best`](Algorithm::best) compares by Deb's rules, as everywhere in genoxide. +/// - **Cost.** The model's fit is O(N³) for N evaluations, its predictions O(N²): Bayesian +/// optimization suits up to a few hundred evaluations of a function that costs far more than +/// that, in up to about 10 to 20 genes. +/// +/// [`model`](Bo::model) gives the model that chose the last point. Every random number comes from +/// streams derived from the [seed](BoBuilder::seed), every operation is a sum, product, quotient +/// or square root in a fixed order with [`math`](crate::math)'s functions, and the multi-starts +/// run on rayon (with the `parallel` feature) with the winner chosen by value, then start: a seed +/// gives the same run on every platform and thread count. +/// +/// Built with [`Bo::builder`], run with an [`Engine`](crate::Engine). +/// +/// ``` +/// use genoxide::prelude::*; +/// use genoxide::problems::{Branin, Problem}; +/// +/// // Branin's function, whose three global minima are 0.397887, in 40 evaluations +/// let bo = Bo::builder(Branin.representation()).minimize().seed(1).build()?; +/// let outcome = Engine::new(bo, Branin) +/// .stop_when(Stop::evaluations(40)) +/// .run()?; +/// assert!(outcome.best_fitness().score().unwrap() < 0.397887 + 1e-3); +/// # Ok::<(), genoxide::Error>(()) +/// ``` +/// +/// References: Močkus, J. (1975). On Bayesian methods for seeking the extremum. *Optimization +/// Techniques IFIP Technical Conference 1974*, LNCS 27: 400-404. Jones, D. R., Schonlau, M. and +/// Welch, W. J. (1998). Efficient global optimization of expensive black-box functions. *Journal +/// of Global Optimization* 13(4): 455-492. Rasmussen, C. E. and Williams, C. K. I. (2006). +/// *Gaussian Processes for Machine Learning.* MIT Press. Ament, S., Daulton, S., Eriksson, D., +/// Balandat, M. and Bakshy, E. (2023). Unexpected improvements to expected improvement for +/// Bayesian optimization. *NeurIPS 2023*, arXiv:2310.20708. McKay, M. D., Beckman, R. J. and +/// Conover, W. J. (1979). A comparison of three methods for selecting values of input variables +/// in the analysis of output from a computer code. *Technometrics* 21(2): 239-245. Loeppky, J. L., +/// Sacks, J. and Welch, W. J. (2009). Choosing the sample size of a computer experiment: a +/// practical guide. *Technometrics* 51(4): 366-376. +#[derive(Clone, Debug)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Bo { + real: Real, + initial_points: usize, + acquisition: Acquisition, + kernel: Kernel, + noise: Noise, + output: Output, + raw_samples: usize, + acquisition_starts: usize, + hyperparameter_starts: usize, + objective: Objective, + seed: u64, + // the initial design, asked in generation 0 + design: Vec, + // every evaluated point, in the order evaluated + observations: Population, + // the logarithms of the last fit's hyperparameters, the next fit's first start + warm: Option>, + // the points of the current ask + pending: Vec>, + indices: Vec, + asked: bool, + reevaluating: bool, + generation: u64, + evaluations: u64, + best: Option>, + best_generation: u64, + // the model that chose the last point, and the best value it was told, in its standardized + // units + #[cfg_attr(feature = "serde", serde(skip))] + model: Option, + #[cfg_attr(feature = "serde", serde(skip))] + model_best: f64, +} + +impl Bo { + /// A builder for Bayesian optimization on `real`. + pub fn builder(real: Real) -> BoBuilder { + BoBuilder { + real, + initial_points: None, + initial_genomes: Vec::new(), + acquisition: Acquisition::default(), + kernel: Kernel::default(), + noise: Noise::default(), + output: Output::default(), + raw_samples: 1000, + acquisition_starts: 10, + hyperparameter_starts: 5, + objective: Objective::default(), + seed: None, + } + } + + /// The representation. + pub fn real(&self) -> &Real { + &self.real + } + + /// The number of points of the initial design. + pub fn initial_points(&self) -> usize { + self.initial_points + } + + /// The acquisition function. + pub fn acquisition(&self) -> Acquisition { + self.acquisition + } + + /// Changes the acquisition function from the next point on, e.g. UCB's β on a schedule from + /// [`Engine::control`](crate::Engine::control). + /// + /// # Errors + /// + /// [`Error::InvalidSetting`] for a negative or non-finite ξ or β. Nothing changes on errors. + pub fn set_acquisition(&mut self, acquisition: Acquisition) -> Result<()> { + acquisition.validate()?; + self.acquisition = acquisition; + Ok(()) + } + + /// The model's kernel. + pub fn kernel(&self) -> Kernel { + self.kernel + } + + /// The model's noise. + pub fn noise(&self) -> Noise { + self.noise + } + + /// What the model fits. + pub fn output(&self) -> Output { + self.output + } + + /// The random points at which the acquisition function is evaluated before its maximization. + pub fn raw_samples(&self) -> usize { + self.raw_samples + } + + /// The best raw samples from which the acquisition function is maximized. + pub fn acquisition_starts(&self) -> usize { + self.acquisition_starts + } + + /// The starts of the maximization of the model's likelihood at each fit. + pub fn hyperparameter_starts(&self) -> usize { + self.hyperparameter_starts + } + + /// The seed of the random numbers: the given one, or a random one if none was given. + pub fn seed(&self) -> u64 { + self.seed + } + + /// The Gaussian process that chose the last point asked, fitted to the evaluations before it: + /// a model of the values the search minimizes (the scores, negated when maximizing, through + /// the [`Output`] transform). `None` before the first point after the initial design, after a + /// random point (when no evaluation had a valid score), and after loading a checkpoint until + /// the next point is asked. + pub fn model(&self) -> Option<&GaussianProcess> { + self.model.as_ref() + } + + /// The acquisition function at `genome` under [`model`](Bo::model), as the search maximizes + /// it: the log expected improvement, the expected improvement, the logarithm of the + /// probability of improvement, or the negated lower confidence bound, in the model's + /// standardized units. `None` without a model. + /// + /// # Panics + /// + /// If `genome` doesn't have a value per gene. + pub fn acquisition_at(&self, genome: &[f64]) -> Option { + let model = self.model.as_ref()?; + assert_eq!( + genome.len(), + self.real.genome_len(), + "a genome of {} genes", + self.real.genome_len() + ); + let scaling = Scaling::new(&self.real); + let mut unit = vec![0.0; scaling.dims()]; + scaling.to_unit(genome, &mut unit); + Some(self.search(model).value(&unit, None)) + } + + fn search<'a>(&self, model: &'a GaussianProcess) -> Search<'a> { + Search { + model, + best: self.model_best, + acquisition: self.acquisition, + scale: model.standardization().1, + } + } + + // the value the model fits for a fitness, before the output transform: the score to minimize, + // or None for an invalid fitness or a score that isn't finite + fn model_value(&self, fitness: Option) -> Option { + let score = fitness?.score()?; + let value = match self.objective { + Objective::Minimize => score, + Objective::Maximize => -score, + }; + value.is_finite().then_some(value) + } + + // whether `genome` was evaluated already + fn observed(&self, genome: &[f64]) -> bool { + self.observations + .iter() + .any(|individual| individual.genome()[..] == genome[..]) + } + + // a random point of the box that wasn't evaluated, from the stream of this generation + fn random_point(&self, generation: u64) -> Reals { + let mut rng = StreamRng::seed_from_u64(self.seed) + .derive(streams::RANDOM) + .derive(generation); + loop { + let genome = self.real.random_genome(&mut rng); + if !self.observed(&genome) { + return genome; + } + } + } + + // the point to evaluate next: the model fitted, the acquisition maximized + fn next_point(&mut self) -> Reals { + let generation = self.generation + 1; + let root = StreamRng::seed_from_u64(self.seed); + let scaling = Scaling::new(&self.real); + let dims = scaling.dims(); + let count = self.observations.len(); + let mut x = vec![0.0; count * dims]; + let mut values = Vec::with_capacity(count); + for (individual, unit) in self.observations.iter().zip(x.chunks_exact_mut(dims)) { + scaling.to_unit(individual.genome(), unit); + values.push(self.model_value(individual.fitness())); + } + self.model = None; + let Some(targets) = transform(&values, self.output) else { + return self.random_point(generation); + }; + let settings = gp::Settings { + kernel: self.kernel, + noise: self.noise, + starts: self.hyperparameter_starts, + seed: root + .derive(streams::HYPERPARAMETERS) + .derive(generation) + .next_u64(), + }; + let model = + GaussianProcess::fit_unit(settings, scaling.clone(), x, &targets, self.warm.as_deref()); + self.warm = Some(model.log_parameters().to_vec()); + // the best point evaluated, by the model's values: the incumbent, and a start + let mut incumbent: Option = None; + for (index, value) in values.iter().enumerate() { + if value.is_some() && incumbent.is_none_or(|best| targets[index] < targets[best]) { + incumbent = Some(index); + } + } + let incumbent = incumbent.unwrap_or(0); + let (y_mean, y_scale) = model.standardization(); + self.model_best = (targets[incumbent] - y_mean) / y_scale; + let search = self.search(&model); + // the raw samples, and the best of them as starts after the incumbent + let mut rng = root.derive(streams::RAW_SAMPLES).derive(generation); + let raw: Vec = (0..self.raw_samples * dims) + .map(|_| rng.unit_f64()) + .collect(); + let raw_values = map_in_order(self.raw_samples, |i| { + search.value(&raw[i * dims..(i + 1) * dims], None) + }); + let mut order: Vec = (0..self.raw_samples).collect(); + // highest first, NaN last; the sort is stable, so ties keep the earlier sample + let key = |v: f64| if v.is_nan() { f64::NEG_INFINITY } else { v }; + order.sort_by(|&a, &b| key(raw_values[b]).total_cmp(&key(raw_values[a]))); + let mut starts = Vec::with_capacity(self.acquisition_starts + 1); + starts.push(model.unit_point(incumbent).to_vec()); + for &i in order.iter().take(self.acquisition_starts) { + starts.push(raw[i * dims..(i + 1) * dims].to_vec()); + } + let unit_box = Real::uniform(dims, 0.0..=1.0).expect("a searched gene"); + let results = map_in_order(starts.len(), |start| { + Lbfgsb::minimize_with( + unit_box.clone(), + Reals::from(starts[start].clone()), + ACQUISITION_EVALUATIONS, + |u, gradient| { + let value = search.value(u, Some(gradient)); + for g in gradient.iter_mut() { + *g = -*g; + } + -value + }, + ) + }); + let mut candidates: Vec<(f64, Reals)> = results + .into_iter() + .flatten() + .map(|(point, value)| (-value, point)) + .collect(); + // highest first; stable, so ties keep the earlier start + candidates.sort_by(|a, b| b.0.total_cmp(&a.0)); + self.model = Some(model); + for (_, unit) in &candidates { + let genome = scaling.to_genome(unit); + if !self.observed(&genome) { + return genome; + } + } + self.random_point(generation) + } + + // the points of the next ask + fn build_ask(&mut self) { + self.pending.clear(); + if self.reevaluating { + let genomes = self.observations.iter().map(|o| o.genome().clone()); + self.pending.extend(genomes.map(Individual::new)); + } else if self.observations.is_empty() { + let design = self.design.iter().cloned(); + self.pending.extend(design.map(Individual::new)); + } else { + let point = self.next_point(); + self.pending.push(Individual::new(point)); + } + self.indices.clear(); + self.indices.extend(0..self.pending.len()); + } + + /// Marks every evaluated point as not evaluated, for a fitness function that changed during + /// the run: the next [`ask`](Algorithm::ask) gives all of them again, and its + /// [`tell`](Algorithm::tell) replaces their values. It isn't a generation; the evaluations + /// are counted. [`best`](Algorithm::best) is then the best of the new values, found in the + /// current generation. No random number is drawn. Before the first tell it changes nothing. + /// + /// # Errors + /// + /// [`Error::ReevaluationOutOfTurn`] between an ask and its tell. Nothing changes on errors. + pub fn reevaluate(&mut self) -> Result<()> { + if self.asked { + return Err(Error::ReevaluationOutOfTurn); + } + if !self.observations.is_empty() { + self.reevaluating = true; + } + Ok(()) + } +} + +// the model's targets from the values to minimize (None for invalid ones): transformed, and the +// invalid ones at the worst of the others; None without a valid value +fn transform(values: &[Option], output: Output) -> Option> { + let mut best = f64::INFINITY; + for value in values.iter().flatten() { + best = best.min(*value); + } + if best == f64::INFINITY { + return None; + } + let offset = match output { + Output::Standardize => 0.0, + Output::Log => log_offset(values, best), + }; + let apply = |value: f64| match output { + Output::Standardize => value, + Output::Log => crate::math::ln(value - best + offset), + }; + let mut worst = f64::NEG_INFINITY; + let mut targets: Vec = values + .iter() + .map(|value| match value { + Some(value) => { + let target = apply(*value); + worst = worst.max(target); + target + } + None => f64::NAN, + }) + .collect(); + for target in &mut targets { + if target.is_nan() { + *target = worst; + } + } + Some(targets) +} + +// δ of the log transform: the first quartile of the distances above the best, the ⌊N/4⌋-th +// smallest of the N valid values' (the best's own 0 counted); the smallest positive one if that's +// 0, and 1 if every value is the best +fn log_offset(values: &[Option], best: f64) -> f64 { + let mut distances: Vec = values.iter().flatten().map(|value| value - best).collect(); + distances.sort_by(f64::total_cmp); + let quartile = distances[distances.len() / 4]; + if quartile > 0.0 { + return quartile; + } + distances + .into_iter() + .find(|&distance| distance > 0.0) + .unwrap_or(1.0) +} + +// the acquisition function of a model, in its standardized units, as maximized +struct Search<'a> { + model: &'a GaussianProcess, + // the best value, standardized + best: f64, + acquisition: Acquisition, + // the model's scale of the values, for ξ + scale: f64, +} + +impl Search<'_> { + // the acquisition's value at the unit-cube point `u`, and its gradient + fn value(&self, u: &[f64], gradient: Option<&mut [f64]>) -> f64 { + let dims = u.len(); + let (mut dmean, mut dvariance) = (Vec::new(), Vec::new()); + let gradients = if gradient.is_some() { + dmean.resize(dims, 0.0); + dvariance.resize(dims, 0.0); + Some((&mut dmean[..], &mut dvariance[..])) + } else { + None + }; + let (mean, variance) = self.model.predict_unit(u, gradients); + let floored = variance < VARIANCE_FLOOR; + let sd = variance.max(VARIANCE_FLOOR).sqrt(); + let [value, by_mean, by_sd] = match self.acquisition { + Acquisition::ExpectedImprovement => { + acquisition::expected_improvement_derivatives(mean, sd, self.best) + } + Acquisition::LogExpectedImprovement => { + acquisition::log_expected_improvement_derivatives(mean, sd, self.best) + } + Acquisition::ProbabilityOfImprovement { xi } => { + acquisition::log_probability_of_improvement_derivatives( + mean, + sd, + self.best, + xi / self.scale, + ) + } + Acquisition::UpperConfidenceBound { beta } => { + acquisition::upper_confidence_bound_derivatives(mean, sd, beta) + } + }; + if let Some(gradient) = gradient { + for i in 0..dims { + let dsd = if floored { + 0.0 + } else { + dvariance[i] / (2.0 * sd) + }; + gradient[i] = by_mean * dmean[i] + by_sd * dsd; + } + } + value + } +} + +impl Reevaluate for Bo { + /// As [`Bo::reevaluate`]: the next ask gives every evaluated point again. + fn reevaluate(&mut self) -> Result<()> { + Bo::reevaluate(self) + } +} + +impl Algorithm for Bo { + type Genome = Reals; + + fn objective(&self) -> Objective { + self.objective + } + + fn ask(&mut self) -> Candidates<'_, Reals> { + if !self.asked { + self.build_ask(); + self.asked = true; + } + Candidates::new(&self.pending, &self.indices) + } + + fn tell(&mut self, fitness: &[Fitness]) -> Result<()> { + if !self.asked { + return Err(Error::TellWithoutAsk); + } + if fitness.len() != self.pending.len() { + return Err(Error::FitnessCount { + expected: self.pending.len(), + got: fitness.len(), + }); + } + self.asked = false; + self.evaluations += fitness.len() as u64; + self.indices.clear(); + if self.reevaluating { + self.reevaluating = false; + self.pending.clear(); + for (observation, &fitness) in self.observations.iter_mut().zip(fitness) { + observation.set_fitness(fitness); + } + self.best = None; + update_best( + &mut self.best, + &mut self.best_generation, + self.generation, + self.objective, + self.observations.as_slice(), + ); + return Ok(()); + } + if !self.observations.is_empty() { + self.generation += 1; + } + let first = self.observations.len(); + for (mut individual, &fitness) in self.pending.drain(..).zip(fitness) { + individual.set_fitness(fitness); + self.observations.push(individual); + } + update_best( + &mut self.best, + &mut self.best_generation, + self.generation, + self.objective, + &self.observations.as_slice()[first..], + ); + Ok(()) + } + + fn population(&self) -> &Population { + &self.observations + } + + fn best(&self) -> Option<&Individual> { + self.best.as_ref() + } + + fn generation(&self) -> u64 { + self.generation + } + + fn evaluations(&self) -> u64 { + self.evaluations + } + + fn best_generation(&self) -> u64 { + self.best_generation + } +} + +// the best so far, from the evaluated `individuals` in order: the first on ties +fn update_best( + best: &mut Option>, + best_generation: &mut u64, + generation: u64, + objective: Objective, + individuals: &[Individual], +) { + for individual in individuals { + let fitness = individual.fitness().unwrap_or(Fitness::invalid()); + let better = match best { + Some(best) => { + objective.is_better(fitness, best.fitness().unwrap_or(Fitness::invalid())) + } + None => true, + }; + if better { + *best = Some(individual.clone()); + *best_generation = generation; + } + } +} + +/// A builder for a [`Bo`], from [`Bo::builder`]. +/// +/// Defaults: maximize; an initial design of 2(n + 1) points for n searched genes; +/// [`Acquisition::LogExpectedImprovement`]; the [Matérn 5/2 kernel](Kernel::Matern52) with +/// [learned noise](Noise::Learned) of at least 1e-6 of the values' variance; +/// [`Output::Standardize`]; 1000 raw samples and 10 starts for the acquisition's maximization; +/// 5 starts for the hyperparameters'; a random seed. +#[derive(Clone, Debug)] +pub struct BoBuilder { + real: Real, + initial_points: Option, + initial_genomes: Vec, + acquisition: Acquisition, + kernel: Kernel, + noise: Noise, + output: Output, + raw_samples: usize, + acquisition_starts: usize, + hyperparameter_starts: usize, + objective: Objective, + seed: Option, +} + +impl BoBuilder { + /// The number of points of the initial design, at least 1: 2(n + 1) for n searched genes + /// (genes whose bounds differ) by default, a small design that leaves most evaluations to the + /// model. 10n is the usual size for an accurate model of the whole box (Loeppky, Sacks and + /// Welch, 2009), more than finding the minimum needs. + pub fn initial_points(mut self, points: usize) -> Self { + self.initial_points = Some(points); + self + } + + /// Genomes to evaluate first, in the initial design: at most + /// [`initial_points`](BoBuilder::initial_points), and distinct. A Latin hypercube sample + /// fills the rest of the design. + pub fn initial_genomes>(mut self, genomes: I) -> Self { + self.initial_genomes = genomes.into_iter().collect(); + self + } + + /// The acquisition function: [`Acquisition::LogExpectedImprovement`] by default. + pub fn acquisition(mut self, acquisition: Acquisition) -> Self { + self.acquisition = acquisition; + self + } + + /// The model's kernel: [`Kernel::Matern52`] by default. + pub fn kernel(mut self, kernel: Kernel) -> Self { + self.kernel = kernel; + self + } + + /// The model's observation noise: [`Noise::Learned`] with a least variance of 1e-6 of the + /// values' variance by default. + pub fn noise(mut self, noise: Noise) -> Self { + self.noise = noise; + self + } + + /// What the model fits: [`Output::Standardize`] by default; [`Output::Log`] for objectives + /// that span orders of magnitude. + pub fn output(mut self, output: Output) -> Self { + self.output = output; + self + } + + /// The random points at which the acquisition function is evaluated before its + /// maximization, at least 1: 1000 by default. They cost a prediction each, O(N²) for N + /// evaluations, far less than the fit. + pub fn raw_samples(mut self, samples: usize) -> Self { + self.raw_samples = samples; + self + } + + /// The best raw samples from which L-BFGS-B maximizes the acquisition function, besides the + /// best point evaluated: 10 by default, at least 1 and at most the raw samples. + pub fn acquisition_starts(mut self, starts: usize) -> Self { + self.acquisition_starts = starts; + self + } + + /// The starts of the maximization of the model's likelihood at each fit, at least 1: 5 by + /// default, the last fit's hyperparameters and random points. + pub fn hyperparameter_starts(mut self, starts: usize) -> Self { + self.hyperparameter_starts = starts; + self + } + + /// Whether higher or lower fitness is better. Maximize by default. + pub fn objective(mut self, objective: Objective) -> Self { + self.objective = objective; + self + } + + /// Higher fitness is better (the default). + pub fn maximize(self) -> Self { + self.objective(Objective::Maximize) + } + + /// Lower fitness is better. + pub fn minimize(self) -> Self { + self.objective(Objective::Minimize) + } + + /// The seed of the random numbers, for a reproducible run: the design and the starts of both + /// maximizations. Random by default. + pub fn seed(mut self, seed: u64) -> Self { + self.seed = Some(seed); + self + } + + /// Validates the settings and creates the search. + /// + /// # Errors + /// + /// - [`Error::InvalidSetting`] for a representation without a gene that has more than one + /// value, an initial design of 0 points or above 2^24, more initial genomes than initial + /// points or the same genome twice, an invalid [`Acquisition`] parameter or [`Noise`], 0 + /// raw samples, acquisition starts of 0 or more than the raw samples, or hyperparameter + /// starts of 0. + /// - [`Error::InvalidGenome`] for an initial genome that doesn't fit the representation. + pub fn build(self) -> Result { + let invalid = + |setting: &'static str, reason: String| Err(Error::InvalidSetting { setting, reason }); + let dims = self.real.variable_genes().len(); + if dims == 0 { + return invalid( + "real", + "Bayesian optimization needs a gene with more than one value".to_string(), + ); + } + let initial_points = self.initial_points.unwrap_or(2 * (dims + 1)); + if initial_points == 0 { + return invalid("initial_points", "must be at least 1, got 0".to_string()); + } + crate::operator::check_size("initial_points", initial_points)?; + if self.initial_genomes.len() > initial_points { + return invalid( + "initial_genomes", + format!( + "at most the initial points, {initial_points}, got {}", + self.initial_genomes.len() + ), + ); + } + for (index, genome) in self.initial_genomes.iter().enumerate() { + self.real.validate(genome)?; + if self.initial_genomes[..index] + .iter() + .any(|other| other[..] == genome[..]) + { + return invalid( + "initial_genomes", + format!("genome {index} is given twice: {genome:?}"), + ); + } + } + self.acquisition.validate()?; + self.noise.validate()?; + if self.raw_samples == 0 { + return invalid("raw_samples", "must be at least 1, got 0".to_string()); + } + crate::operator::check_size("raw_samples", self.raw_samples)?; + if self.acquisition_starts == 0 || self.acquisition_starts > self.raw_samples { + return invalid( + "acquisition_starts", + format!( + "must be at least 1 and at most the raw samples, {}, got {}", + self.raw_samples, self.acquisition_starts + ), + ); + } + if self.hyperparameter_starts == 0 { + return invalid( + "hyperparameter_starts", + "must be at least 1, got 0".to_string(), + ); + } + crate::operator::check_size("hyperparameter_starts", self.hyperparameter_starts)?; + let seed = self + .seed + .unwrap_or_else(|| StreamRng::from_entropy().next_u64()); + let mut design = self.initial_genomes; + let missing = initial_points - design.len(); + if missing > 0 { + let mut rng = StreamRng::seed_from_u64(seed).derive(streams::DESIGN); + design.extend(self.real.latin_hypercube(missing, &mut rng)?); + } + Ok(Bo { + real: self.real, + initial_points, + acquisition: self.acquisition, + kernel: self.kernel, + noise: self.noise, + output: self.output, + raw_samples: self.raw_samples, + acquisition_starts: self.acquisition_starts, + hyperparameter_starts: self.hyperparameter_starts, + objective: self.objective, + seed, + design, + observations: Population::default(), + warm: None, + pending: Vec::new(), + indices: Vec::new(), + asked: false, + reevaluating: false, + generation: 0, + evaluations: 0, + best: None, + best_generation: 0, + model: None, + model_best: f64::NAN, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // a model of a smooth function of 2 genes, from 10 points + fn model() -> (GaussianProcess, f64) { + let real = Real::new([-1.0..=2.0, 0.0..=3.0]).unwrap(); + let points = real + .latin_hypercube(10, &mut StreamRng::seed_from_u64(2)) + .unwrap(); + let values: Vec = points + .iter() + .map(|x| (2.0 * x[0]).sin() + 0.5 * x[1] * x[1] - 0.3 * x[0] * x[1]) + .collect(); + let scaling = Scaling::new(&real); + let mut x = vec![0.0; 20]; + for (point, unit) in points.iter().zip(x.as_chunks_mut::<2>().0) { + scaling.to_unit(point, unit); + } + let settings = gp::Settings { + kernel: Kernel::Matern52, + noise: Noise::default(), + starts: 3, + seed: 1, + }; + let model = GaussianProcess::fit_unit(settings, scaling, x, &values, None); + let (mean, scale) = model.standardization(); + let best = values.iter().fold(f64::INFINITY, |a, &b| a.min(b)); + (model, (best - mean) / scale) + } + + #[test] + fn the_acquisitions_gradients_match_central_differences() { + let (model, best) = model(); + let acquisitions = [ + Acquisition::ExpectedImprovement, + Acquisition::LogExpectedImprovement, + Acquisition::ProbabilityOfImprovement { xi: 0.01 }, + Acquisition::UpperConfidenceBound { beta: 2.0 }, + ]; + for acquisition in acquisitions { + let search = Search { + model: &model, + best, + acquisition, + scale: model.standardization().1, + }; + for u in [[0.3, 0.6], [0.9, 0.1], [0.55, 0.45], [0.05, 0.95]] { + let mut gradient = [0.0; 2]; + let value = search.value(&u, Some(&mut gradient)); + assert_eq!(value.to_bits(), search.value(&u, None).to_bits()); + for i in 0..2 { + let h = 1e-6; + let (mut plus, mut minus) = (u, u); + plus[i] += h; + minus[i] -= h; + let numeric = + (search.value(&plus, None) - search.value(&minus, None)) / (2.0 * h); + assert!( + (gradient[i] - numeric).abs() <= 1e-5 * numeric.abs().max(1.0), + "{acquisition:?} at {u:?}, gene {i}: {} against {numeric}", + gradient[i] + ); + } + } + } + } + + #[test] + fn the_log_transform_keeps_the_order_and_imputes_the_worst() { + let values = [Some(3.0), None, Some(10.0), Some(3.5), Some(1e6), Some(4.0)]; + let targets = transform(&values, Output::Log).unwrap(); + // δ is the first quartile of the distances above the best, 0.5 here: the ⌊5/4⌋-th of + // 0, 0.5, 1, 7 and 999997 + assert_eq!(targets[0], 0.5_f64.ln()); + assert_eq!(targets[3], 1.0_f64.ln()); + assert!(targets[0] < targets[3] && targets[3] < targets[5] && targets[5] < targets[2]); + assert_eq!(targets[1], targets[4]); + let targets = transform(&values, Output::Standardize).unwrap(); + assert_eq!(targets, [3.0, 1e6, 10.0, 3.5, 1e6, 4.0]); + assert_eq!(transform(&[None, None], Output::Log), None); + // every value the best: δ is 1 + assert_eq!( + transform(&[Some(2.0), Some(2.0)], Output::Log), + Some(vec![0.0, 0.0]) + ); + } +} diff --git a/src/algorithm/bo/acquisition.rs b/src/algorithm/bo/acquisition.rs index 822e825a..75f08fbf 100644 --- a/src/algorithm/bo/acquisition.rs +++ b/src/algorithm/bo/acquisition.rs @@ -130,6 +130,78 @@ pub fn upper_confidence_bound(mean: f64, sd: f64, beta: f64, objective: Objectiv mean + beta.sqrt() * sd } +// The functions as Bayesian optimization maximizes them, with their derivatives with respect to +// the mean and the standard deviation, `[value, ∂/∂μ, ∂/∂σ]`, when minimizing, for σ > 0. + +// the expected improvement: ∂EI/∂μ = −Φ(z), ∂EI/∂σ = φ(z) +pub(crate) fn expected_improvement_derivatives(mean: f64, sd: f64, best: f64) -> [f64; 3] { + let z = (best - mean) / sd; + let (pdf, cdf) = (normal_pdf(z), normal_cdf(z)); + [(sd * (pdf + z * cdf)).max(0.0), -cdf, pdf] +} + +// the log expected improvement: ln σ + log_h(z), with log_h′(z) = Φ(z)/h(z), h = φ + zΦ, so +// ∂/∂μ = −log_h′(z)/σ and ∂/∂σ = (1 − z log_h′(z))/σ +pub(crate) fn log_expected_improvement_derivatives(mean: f64, sd: f64, best: f64) -> [f64; 3] { + let z = (best - mean) / sd; + let slope = log_h_derivative(z); + [ln(sd) + log_h(z), -slope / sd, (1.0 - z * slope) / sd] +} + +// the logarithm of the probability of improvement by ξ, ln Φ(z), z = (best − μ − ξ)/σ: the same +// maximizer as Φ(z), and a gradient where Φ underflows; d ln Φ/dz = φ(z)/Φ(z) +pub(crate) fn log_probability_of_improvement_derivatives( + mean: f64, + sd: f64, + best: f64, + xi: f64, +) -> [f64; 3] { + let z = (best - mean - xi) / sd; + let (value, slope) = log_normal_cdf(z); + [value, -slope / sd, -slope * z / sd] +} + +// the upper confidence bound when minimizing, −μ + √β σ +pub(crate) fn upper_confidence_bound_derivatives(mean: f64, sd: f64, beta: f64) -> [f64; 3] { + let root = beta.sqrt(); + [-mean + root * sd, -1.0, root] +} + +// log_h′(z) = Φ(z) / (φ(z) + z Φ(z)). Below −1 through the Mills ratio m = Φ/φ = +// √(π/2) erfcx(−z/√2), as m / (1 + z m), with 1 + z m = 1 − e^(ln(|z| m)) from the same +// `log1mexp` argument as `log_h`. That difference loses ε z² of relative accuracy, so below −64 +// the asymptotic series −z (1 + 2a − 6a² + 42a³), a = 1/z², from the Mills ratio's series +// m = (1 − a + 3a² − 15a³ + …)/|z| (Abramowitz and Stegun, 7.1.23), whose next term is about +// 414a⁴ (both checked against mpmath): within 2e-12 either way at −64. +fn log_h_derivative(z: f64) -> f64 { + if z > -1.0 { + let cdf = normal_cdf(z); + cdf / (normal_pdf(z) + z * cdf) + } else if z > -64.0 { + let t = erfcx(-z * FRAC_1_SQRT_2); + let mills = SQRT_PI_OVER_2 * t; + let x = ln(t * -z) + C2; + mills / -exp_m1(x) + } else { + let a = 1.0 / (z * z); + -z * (1.0 + a * (2.0 + a * (-6.0 + 42.0 * a))) + } +} + +// ln Φ(z) and φ(z)/Φ(z): below −1 through Φ(z) = erfcx(−z/√2) e^(−z²/2) / 2 +fn log_normal_cdf(z: f64) -> (f64, f64) { + if z > -1.0 { + let cdf = normal_cdf(z); + (ln(cdf), normal_pdf(z) / cdf) + } else { + let t = erfcx(-z * FRAC_1_SQRT_2); + (ln(0.5 * t) - 0.5 * z * z, 1.0 / (SQRT_PI_OVER_2 * t)) + } +} + +// √(π/2) +const SQRT_PI_OVER_2: f64 = 1.253_314_137_315_500_3; + // a negative or NaN standard deviation or parameter fn invalid(x: f64) -> bool { x.is_nan() || x < 0.0 @@ -341,6 +413,93 @@ mod tests { assert!(upper_confidence_bound(1.0, 2.0, -1.0, Maximize).is_nan()); } + #[test] + fn derivatives_match_central_differences_and_the_values() { + type Derivatives = fn(f64, f64) -> [f64; 3]; + let functions: [(&str, Derivatives); 4] = [ + ("EI", |mean, sd| { + expected_improvement_derivatives(mean, sd, 1.0) + }), + ("log-EI", |mean, sd| { + log_expected_improvement_derivatives(mean, sd, 1.0) + }), + ("log-PI", |mean, sd| { + log_probability_of_improvement_derivatives(mean, sd, 1.0, 0.05) + }), + ("UCB", |mean, sd| { + upper_confidence_bound_derivatives(mean, sd, 4.0) + }), + ]; + // z from about −60 to 3, in every range of log_h + for (name, f) in functions { + for (mean, sd) in [ + (0.5, 0.3), + (1.0, 1.0), + (1.3, 0.2), + (2.2, 0.5), + (4.0, 0.1), + (7.0, 0.1), + ] { + let [value, by_mean, by_sd] = f(mean, sd); + let h = 1e-6; + let numeric_mean = (f(mean + h, sd)[0] - f(mean - h, sd)[0]) / (2.0 * h); + let numeric_sd = (f(mean, sd + h)[0] - f(mean, sd - h)[0]) / (2.0 * h); + for (analytic, numeric) in [(by_mean, numeric_mean), (by_sd, numeric_sd)] { + assert!( + (analytic - numeric).abs() <= 1e-6 * analytic.abs().max(1.0), + "{name} at μ = {mean}, σ = {sd} ({value}): {analytic} against {numeric}" + ); + } + } + } + // the values are the public functions', minimizing + for (mean, sd) in [(0.5, 0.3), (2.2, 0.5), (40.0, 0.5)] { + let objective = Minimize; + let ei = expected_improvement(mean, sd, 1.0, objective); + assert_eq!(expected_improvement_derivatives(mean, sd, 1.0)[0], ei); + let log_ei = log_expected_improvement(mean, sd, 1.0, objective); + assert_eq!( + log_expected_improvement_derivatives(mean, sd, 1.0)[0], + log_ei + ); + let pi = probability_of_improvement(mean, sd, 1.0, 0.05, objective); + let log_pi = log_probability_of_improvement_derivatives(mean, sd, 1.0, 0.05)[0]; + if pi > 0.0 { + assert!((log_pi - ln(pi)).abs() <= 1e-12 * log_pi.abs().max(1.0)); + } else { + // where the probability underflows, its logarithm stays finite + assert!(log_pi.is_finite() && log_pi < -700.0); + } + let ucb = upper_confidence_bound(mean, sd, 4.0, objective); + assert_eq!(upper_confidence_bound_derivatives(mean, sd, 4.0)[0], ucb); + } + // log_h′(z) = Φ(z) / (φ(z) + z Φ(z)) in each range, against mpmath (to 60 digits, + // rounded): within 4e-12, relative, at either side of the switch to the series at −64 + let reference = [ + (-0.5, 1.5598731483480797), + (-1.5, 2.2795806941564463), + (-10.0, 10.194383033412553), + (-63.9, 63.93127594805792), + (-64.1, 64.13117850553884), + (-100.0, 100.01999400419587), + (-1000.0, 1000.001999994), + (-1e6, 1000000.000002), + ]; + for (z, expected) in reference { + let slope = log_h_derivative(z); + assert!( + (slope - expected).abs() <= 4e-12 * expected, + "z = {z}: {slope} against {expected}" + ); + } + // far below the best, where the expected improvement underflows: finite slopes + for mean in [1e3, 1e9, 1e150] { + let [value, by_mean, by_sd] = log_expected_improvement_derivatives(mean, 1.0, 0.0); + assert!(value.is_finite() && by_mean.is_finite() && by_sd.is_finite()); + assert!(by_mean < 0.0 && by_sd > 0.0); + } + } + #[test] fn portable_values() { let values = [ diff --git a/src/prelude.rs b/src/prelude.rs index c94ba515..8503fb8f 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -4,6 +4,7 @@ //! use genoxide::prelude::*; //! ``` +pub use crate::algorithm::bo::{self, Bo}; pub use crate::algorithm::cmaes::{self, Cmaes}; pub use crate::algorithm::continuation::{self, Continuation, Continue}; pub use crate::algorithm::de::{self, De}; diff --git a/tests/algorithms.rs b/tests/algorithms.rs index 95e96fd6..2d53446e 100644 --- a/tests/algorithms.rs +++ b/tests/algorithms.rs @@ -416,8 +416,20 @@ fn portable_runs() { .build() .unwrap(), ), + // the Gaussian process's fits, the acquisition's maximization, the Latin hypercube + portable_run(Bo::builder(real()).minimize().seed(1).build().unwrap()), + portable_run( + Bo::builder(real()) + .output(bo::Output::Log) + .acquisition(bo::Acquisition::ProbabilityOfImprovement { xi: 0.1 }) + .kernel(genoxide::model::gp::Kernel::SquaredExponential) + .minimize() + .seed(1) + .build() + .unwrap(), + ), ]; - let expected: [[f64; 4]; 14] = [ + let expected: [[f64; 4]; 16] = [ // L-SHADE [ 0.5886518163542276, @@ -515,6 +527,19 @@ fn portable_runs() { -2.7974966575791904, 1.1930722176383557, -1.3732603836128217, + ], // Bayesian optimization: log-EI, Matérn 5/2 + [ + 0.48198052392264934, + 0.055399166715114134, + 0.7923531293628381, + 0.5613182693321956, + ], + // Bayesian optimization: the log transform, PI, the squared exponential + [ + 1.1457818657817844, + 1.012510092068803, + 0.8225014439106566, + 0.8156551023913199, ], ]; for (run, expected) in runs.iter().zip(expected) { diff --git a/tests/bo.rs b/tests/bo.rs new file mode 100644 index 00000000..bdd19d7c --- /dev/null +++ b/tests/bo.rs @@ -0,0 +1,386 @@ +//! Bayesian optimization: the classic problems of Jones, Schonlau and Welch (1998) reached in tens +//! of evaluations, the guarantees of every algorithm, and how invalid points are handled. + +use genoxide::algorithm::bo::{Acquisition, Output}; +use genoxide::model::gp::Kernel; +use genoxide::prelude::*; +use genoxide::problems::{Branin, GoldsteinPrice, Hartmann3, Problem, SixHumpCamel}; + +// the evaluations a run took to come within 1e-3 of `optimum`, or None +fn evaluations_to_target( + bo: Bo, + fitness: impl FitnessFunction, + optimum: f64, + budget: u64, +) -> Option { + let outcome = Engine::new(bo, fitness) + .stop_when(Stop::target(optimum + 1e-3).or(Stop::evaluations(budget))) + .run() + .unwrap(); + (outcome.stop_reason() == StopReason::Target).then_some(outcome.evaluations()) +} + +#[test] +fn branin_in_tens_of_evaluations() { + // three global minima of 0.397887; a median of 30 evaluations over 20 seeds + for seed in 1..=3 { + let bo = Bo::builder(Branin.representation()) + .minimize() + .seed(seed) + .build() + .unwrap(); + let evaluations = evaluations_to_target(bo, Branin, 0.397_887_357_729_738, 50); + assert!(evaluations.is_some(), "seed {seed}"); + } +} + +#[test] +fn six_hump_camel_in_tens_of_evaluations() { + let optimum = SixHumpCamel.optimum().unwrap().value(); + // the usual box for Bayesian optimization, [−3, 3] × [−2, 2]: a median of 49 evaluations + let narrow = Real::new([-3.0..=3.0, -2.0..=2.0]).unwrap(); + let bo = Bo::builder(narrow).minimize().seed(1).build().unwrap(); + assert!(evaluations_to_target(bo, SixHumpCamel, optimum, 70).is_some()); + // genoxide's box, [−5, 5]², whose corners reach 6,400: the log transform resolves the + // minima where the standardized values can't (a median of 34 evaluations against 6 of 20 + // runs within 80) + let bo = Bo::builder(SixHumpCamel.representation()) + .output(Output::Log) + .minimize() + .seed(1) + .build() + .unwrap(); + assert!(evaluations_to_target(bo, SixHumpCamel, optimum, 60).is_some()); +} + +#[test] +fn hartmann_3_in_tens_of_evaluations() { + // 12 of 20 seeds within 80 evaluations, a median of 24; the others stall where the model + // judges x₁ irrelevant, at x₁ = 0, 0.008 above the minimum + let optimum = Hartmann3.optimum().unwrap().value(); + let bo = Bo::builder(Hartmann3.representation()) + .minimize() + .seed(3) + .build() + .unwrap(); + assert!(evaluations_to_target(bo, Hartmann3, optimum, 50).is_some()); +} + +#[test] +fn goldstein_price_with_the_log_transform() { + // values from 3 to 10⁶: no seed of 20 reaches the target within 80 evaluations with + // standardized values, every one with the log transform + let bo = Bo::builder(GoldsteinPrice.representation()) + .output(Output::Log) + .minimize() + .seed(1) + .build() + .unwrap(); + assert!(evaluations_to_target(bo, GoldsteinPrice, 3.0, 70).is_some()); +} + +#[test] +fn every_acquisition_and_kernel_finds_branin_minima() { + let settings = [ + (Acquisition::ExpectedImprovement, Kernel::Matern52), + ( + Acquisition::ProbabilityOfImprovement { xi: 0.01 }, + Kernel::Matern52, + ), + ( + Acquisition::UpperConfidenceBound { beta: 4.0 }, + Kernel::Matern52, + ), + ( + Acquisition::LogExpectedImprovement, + Kernel::SquaredExponential, + ), + ]; + for (acquisition, kernel) in settings { + let bo = Bo::builder(Branin.representation()) + .acquisition(acquisition) + .kernel(kernel) + .minimize() + .seed(1) + .build() + .unwrap(); + let outcome = Engine::new(bo, Branin) + .stop_when(Stop::evaluations(40)) + .run() + .unwrap(); + let best = outcome.best_fitness().score().unwrap(); + assert!( + best < 0.397_887 + 0.05, + "{acquisition:?}, {kernel:?}: {best}" + ); + } +} + +// the evaluated genomes of a run, in order, and its best +fn run(bo: Bo, parallel: bool, evaluations: u64) -> (Vec>, Individual) { + let mut engine = Engine::new(bo, Branin) + .parallel(parallel) + .stop_when(Stop::evaluations(evaluations)); + let outcome = engine.run().unwrap(); + let genomes = engine + .algorithm() + .population() + .iter() + .map(|individual| individual.genome().iter().map(|x| x.to_bits()).collect()) + .collect(); + (genomes, outcome.into_best()) +} + +fn branin_bo(seed: u64) -> Bo { + Bo::builder(Branin.representation()) + .minimize() + .seed(seed) + .build() + .unwrap() +} + +#[test] +fn reproducible_sequentially_and_in_parallel() { + let (genomes, best) = run(branin_bo(5), false, 20); + assert_eq!( + run(branin_bo(5), false, 20), + (genomes.clone(), best.clone()) + ); + assert_eq!(run(branin_bo(5), true, 20), (genomes.clone(), best)); + assert_ne!(run(branin_bo(6), false, 20).0, genomes); +} + +#[test] +fn no_point_is_asked_twice() { + let (genomes, _) = run(branin_bo(2), false, 40); + assert_eq!(genomes.len(), 40); + for (i, genome) in genomes.iter().enumerate() { + assert!(!genomes[..i].contains(genome), "point {i} is asked again"); + } +} + +#[test] +fn maximizing_mirrors_minimizing() { + let negated = |x: &Reals| -Branin.evaluate(x); + let real = Branin.representation(); + let minimized = Engine::new(branin_bo(4), Branin) + .stop_when(Stop::evaluations(15)) + .run() + .unwrap(); + let maximized = Bo::builder(real).maximize().seed(4).build().unwrap(); + let maximized = Engine::new(maximized, negated) + .stop_when(Stop::evaluations(15)) + .run() + .unwrap(); + assert_eq!(maximized.best_genome(), minimized.best_genome()); + assert_eq!( + maximized.best_fitness().score().unwrap(), + -minimized.best_fitness().score().unwrap() + ); +} + +#[test] +fn the_design_and_the_protocol() { + let real = Real::uniform(3, 0.0..=1.0).unwrap(); + let given = vec![ + Reals::from(vec![0.5, 0.5, 0.5]), + Reals::from(vec![0.1, 0.9, 0.2]), + ]; + let mut bo = Bo::builder(real) + .initial_genomes(given.clone()) + .minimize() + .seed(1) + .build() + .unwrap(); + // 2(n + 1) = 8 points, the given ones first + assert_eq!(bo.initial_points(), 8); + let design: Vec = bo.ask().iter().cloned().collect(); + assert_eq!(design.len(), 8); + assert_eq!(&design[..2], &given[..]); + // asking again gives the same; a tell of the wrong count changes nothing + assert_eq!(bo.ask().iter().cloned().collect::>(), design); + assert!(matches!( + bo.tell(&[Fitness::new(1.0)]), + Err(Error::FitnessCount { + expected: 8, + got: 1 + }) + )); + let sphere = |x: &Reals| Fitness::new(x.iter().map(|v| (v - 0.3) * (v - 0.3)).sum()); + let fitness: Vec = design.iter().map(sphere).collect(); + bo.tell(&fitness).unwrap(); + assert!(matches!(bo.tell(&fitness), Err(Error::TellWithoutAsk))); + assert_eq!((bo.generation(), bo.evaluations()), (0, 8)); + assert!(bo.model().is_none()); + // then one point per generation, chosen by a model + for generation in 1..=3 { + let asked: Vec = bo.ask().iter().cloned().collect(); + assert_eq!(asked.len(), 1); + assert!( + bo.model() + .is_some_and(|model| model.len() == 7 + generation) + ); + assert!(bo.acquisition_at(&asked[0]).unwrap().is_finite()); + bo.tell(&[sphere(&asked[0])]).unwrap(); + assert_eq!(bo.generation(), generation as u64); + } + assert_eq!(bo.population().len(), 11); +} + +#[test] +fn invalid_points_enter_the_model_at_the_worst_value() { + // undefined where x₁ + x₂ > 14: the search goes on around the region + let partial = |x: &Reals| (x[0] + x[1] <= 14.0).then(|| Branin.evaluate(x)); + let bo = branin_bo(1); + let mut engine = Engine::new(bo, partial).stop_when(Stop::evaluations(45)); + let outcome = engine.run().unwrap(); + assert!(outcome.best_fitness().score().unwrap() < 0.397_887 + 1e-2); + let invalid = engine + .algorithm() + .population() + .iter() + .filter(|individual| individual.fitness().unwrap().score().is_none()) + .count(); + assert!(invalid < 10, "{invalid} invalid points"); + // nothing valid at all: random points, never a panic + let nothing = |_: &Reals| None::; + let outcome = Engine::new(branin_bo(1), nothing) + .stop_when(Stop::evaluations(10)) + .run() + .unwrap(); + assert_eq!(outcome.evaluations(), 10); +} + +#[test] +fn reevaluation_asks_every_point_again() { + let mut bo = branin_bo(3); + for _ in 0..3 { + let fitness: Vec = bo + .ask() + .iter() + .map(|x| Fitness::new(Branin.evaluate(x))) + .collect(); + bo.tell(&fitness).unwrap(); + } + let observed: Vec = bo.population().iter().map(|o| o.genome().clone()).collect(); + bo.reevaluate().unwrap(); + let asked: Vec = bo.ask().iter().cloned().collect(); + assert_eq!(asked, observed); + assert!(matches!(bo.reevaluate(), Err(Error::ReevaluationOutOfTurn))); + // a function shifted by 10: the same order, new values + let fitness: Vec = asked + .iter() + .map(|x| Fitness::new(Branin.evaluate(x) + 10.0)) + .collect(); + let generation = bo.generation(); + bo.tell(&fitness).unwrap(); + assert_eq!(bo.generation(), generation); + assert_eq!(bo.best_generation(), generation); + assert!(bo.best().unwrap().fitness().unwrap().score().unwrap() > 10.0); + assert_eq!(bo.ask().len(), 1); +} + +#[test] +fn ucb_beta_changes_during_a_run() { + let bo = Bo::builder(Branin.representation()) + .acquisition(Acquisition::UpperConfidenceBound { beta: 9.0 }) + .minimize() + .seed(1) + .build() + .unwrap(); + let outcome = Engine::new(bo, Branin) + .stop_when(Stop::evaluations(30)) + .control(|bo: &mut Bo, progress| { + let beta = 9.0 / (1.0 + progress.generation() as f64); + bo.set_acquisition(Acquisition::UpperConfidenceBound { beta }) + }) + .run() + .unwrap(); + assert!(outcome.best_fitness().score().unwrap() < 0.5); + let mut bo = branin_bo(1); + assert!(matches!( + bo.set_acquisition(Acquisition::UpperConfidenceBound { beta: -1.0 }), + Err(Error::InvalidSetting { .. }) + )); + assert_eq!(bo.acquisition(), Acquisition::LogExpectedImprovement); +} + +#[test] +fn invalid_settings_are_errors() { + let real = || Real::uniform(2, 0.0..=1.0).unwrap(); + let setting = |result: Result| match result { + Err(Error::InvalidSetting { setting, .. }) => setting, + other => panic!("not an invalid setting: {other:?}"), + }; + let fixed = Real::uniform(2, 1.0..=1.0).unwrap(); + assert_eq!(setting(Bo::builder(fixed).build()), "real"); + assert_eq!( + setting(Bo::builder(real()).initial_points(0).build()), + "initial_points" + ); + let genomes = vec![Reals::from(vec![0.5, 0.5]); 2]; + assert_eq!( + setting(Bo::builder(real()).initial_genomes(genomes.clone()).build()), + "initial_genomes" + ); + assert_eq!( + setting( + Bo::builder(real()) + .initial_points(1) + .initial_genomes(vec![ + Reals::from(vec![0.5, 0.5]), + Reals::from(vec![0.2, 0.5]) + ]) + .build() + ), + "initial_genomes" + ); + assert!(matches!( + Bo::builder(real()) + .initial_genomes(vec![Reals::from(vec![2.0, 0.5])]) + .build(), + Err(Error::InvalidGenome { .. }) + )); + for acquisition in [ + Acquisition::ProbabilityOfImprovement { xi: -0.1 }, + Acquisition::UpperConfidenceBound { beta: f64::NAN }, + ] { + let result = Bo::builder(real()).acquisition(acquisition).build(); + assert_eq!(setting(result), "acquisition"); + } + let noise = genoxide::model::gp::Noise::Learned { min: 0.0 }; + assert_eq!(setting(Bo::builder(real()).noise(noise).build()), "noise"); + assert_eq!( + setting(Bo::builder(real()).raw_samples(0).build()), + "raw_samples" + ); + for starts in [0, 1001] { + let result = Bo::builder(real()).acquisition_starts(starts).build(); + assert_eq!(setting(result), "acquisition_starts"); + } + assert_eq!( + setting(Bo::builder(real()).hyperparameter_starts(0).build()), + "hyperparameter_starts" + ); +} + +#[test] +fn a_fixed_gene_takes_no_part() { + // Branin with a third gene fixed at 7 + let real = Real::new([-5.0..=10.0, 0.0..=15.0, 7.0..=7.0]).unwrap(); + let fitness = |x: &Reals| Branin.evaluate(&Reals::from(x[..2].to_vec())); + let bo = Bo::builder(real).minimize().seed(1).build().unwrap(); + assert_eq!(bo.initial_points(), 6); + let mut engine = Engine::new(bo, fitness).stop_when(Stop::evaluations(40)); + let outcome = engine.run().unwrap(); + assert!(outcome.best_fitness().score().unwrap() < 0.397_887 + 1e-3); + assert!( + engine + .algorithm() + .population() + .iter() + .all(|o| o.genome()[2] == 7.0) + ); + let length_scales = engine.algorithm().model().unwrap().hyperparameters(); + assert_eq!(length_scales.length_scales()[2], f64::INFINITY); +} diff --git a/tests/checkpoint.rs b/tests/checkpoint.rs index 150dd5b0..8f9a603c 100644 --- a/tests/checkpoint.rs +++ b/tests/checkpoint.rs @@ -1016,6 +1016,64 @@ fn an_lbfgsb_saved_between_an_ask_and_its_tell_resumes() { assert_eq!(bytes(&resumed), bytes(&lbfgsb)); } +// Branin's function, for Bayesian optimization +fn branin(x: &Reals) -> f64 { + use genoxide::problems::Branin; + Branin.evaluate(x) +} + +fn bo(seed: u64) -> Bo { + use genoxide::problems::{Branin, Problem}; + Bo::builder(Branin.representation()) + .minimize() + .seed(seed) + .build() + .unwrap() +} + +#[test] +fn bayesian_optimization_resumes() { + // the warm start of the hyperparameters is saved; the model is fitted again on the next ask + resumes(|| bo(1), branin, 4, 12); + let log = || { + use genoxide::problems::{Branin, Problem}; + Bo::builder(Branin.representation()) + .output(bo::Output::Log) + .acquisition(bo::Acquisition::UpperConfidenceBound { beta: 2.0 }) + .minimize() + .seed(2) + .build() + .unwrap() + }; + resumes(log, branin, 0, 8); +} + +#[test] +fn bayesian_optimization_saved_between_an_ask_and_its_tell_resumes() { + let mut bo = bo(3); + for _ in 0..4 { + let fitness: Vec = bo.ask().iter().map(|x| Fitness::new(branin(x))).collect(); + bo.tell(&fitness).unwrap(); + } + let asked: Vec> = bo.ask().iter().map(|x| x.to_vec()).collect(); + let mut resumed: Bo = checkpoint::load(bytes(&bo).as_slice()).unwrap(); + let again: Vec> = resumed.ask().iter().map(|x| x.to_vec()).collect(); + assert_eq!(again, asked); + let fitness: Vec = asked.iter().map(|x| Fitness::new(branin(&Reals::from(x.clone())))).collect(); + bo.tell(&fitness).unwrap(); + resumed.tell(&fitness).unwrap(); + assert_eq!(bytes(&resumed), bytes(&bo)); + for _ in 0..3 { + let a: Vec> = bo.ask().iter().map(|x| x.to_vec()).collect(); + let b: Vec> = resumed.ask().iter().map(|x| x.to_vec()).collect(); + assert_eq!(a, b); + let fitness: Vec = a.iter().map(|x| Fitness::new(branin(&Reals::from(x.clone())))).collect(); + bo.tell(&fitness).unwrap(); + resumed.tell(&fitness).unwrap(); + } + assert_eq!(bytes(&resumed), bytes(&bo)); +} + // a continuation: Adam through 3 stages of a smoothed Σ |xᵢ − cᵢ|, its ε shared by an atomic mod continuation { use super::bytes; From abc8e803dba4dc608ac6ce462736c0fd707f7997 Mon Sep 17 00:00:00 2001 From: tachsin Date: Fri, 2 Oct 2026 12:36:04 +0300 Subject: [PATCH 07/13] feat(model): no noise by default, the model interpolating deterministic functions genoxide's fitness functions are deterministic, and a Gaussian process that interpolates their values resolves the small differences near a minimum that a learned noise smooths over. Measured with Bo over 20 seeds and 80 evaluations to f* + 1e-4: Branin reached in 20 runs, the six-hump camel in [-3, 3] x [-2, 2] in 20 and Hartmann 3 in 20, against 14, 17 and 12 with noise learned from a least of 1e-6, which settled above that least (an absolute standard deviation of about 0.03 on Branin). Hartmann 6 reached its minimum in as many runs, far more precisely. Noise::Learned stays for noisy functions. The documented measurements of the log transform are redone with the new default, and the tests' budgets with them. --- src/algorithm/bo.rs | 19 ++++++++++--------- src/model/gp.rs | 33 ++++++++++++++++++++------------- tests/algorithms.rs | 18 +++++++++--------- tests/bo.rs | 29 ++++++++++++++++------------- tests/checkpoint.rs | 10 ++++++++-- 5 files changed, 63 insertions(+), 46 deletions(-) diff --git a/src/algorithm/bo.rs b/src/algorithm/bo.rs index e22b73f9..e19b0579 100644 --- a/src/algorithm/bo.rs +++ b/src/algorithm/bo.rs @@ -82,10 +82,11 @@ pub enum Output { /// best, so the best point is never an outlier far below the others (as it would be with a /// tiny `δ`), and shrinks as the search closes in. /// - /// Measured on 20 seeds per problem (default settings otherwise, to f* + 1e-3): Goldstein-Price - /// in [−2, 2]² reached in 20 runs of 20 within 80 evaluations (0 with - /// [`Standardize`](Output::Standardize)), the six-hump camel in [−5, 5]² in 20 (6), Branin in - /// a median of 25 evaluations (30). + /// Measured on 20 seeds per problem, with the other settings' defaults, to f* + 1e-3 within 80 + /// evaluations: Goldstein-Price in [−2, 2]² reached in 19 runs (0 with + /// [`Standardize`](Output::Standardize)), the six-hump camel in [−5, 5]² in 20 (4), Branin in + /// a median of 20 evaluations (30). Not for every function: on Hartmann 3, whose values span + /// less than an order of magnitude, 13 runs (20). Log, } @@ -712,9 +713,8 @@ fn update_best( /// A builder for a [`Bo`], from [`Bo::builder`]. /// /// Defaults: maximize; an initial design of 2(n + 1) points for n searched genes; -/// [`Acquisition::LogExpectedImprovement`]; the [Matérn 5/2 kernel](Kernel::Matern52) with -/// [learned noise](Noise::Learned) of at least 1e-6 of the values' variance; -/// [`Output::Standardize`]; 1000 raw samples and 10 starts for the acquisition's maximization; +/// [`Acquisition::LogExpectedImprovement`]; the [Matérn 5/2 kernel](Kernel::Matern52) without +/// noise, the model interpolating the values; [`Output::Standardize`]; 1000 raw samples and 10 starts for the acquisition's maximization; /// 5 starts for the hyperparameters'; a random seed. #[derive(Clone, Debug)] pub struct BoBuilder { @@ -762,8 +762,9 @@ impl BoBuilder { self } - /// The model's observation noise: [`Noise::Learned`] with a least variance of 1e-6 of the - /// values' variance by default. + /// The model's observation noise: none by default ([`Noise::Fixed`] of 0), the model + /// interpolating the values of a deterministic function; [`Noise::Learned`] for a noisy one. + /// See [`Noise`] for the evidence. pub fn noise(mut self, noise: Noise) -> Self { self.noise = noise; self diff --git a/src/model/gp.rs b/src/model/gp.rs index 375e3e64..e67b4099 100644 --- a/src/model/gp.rs +++ b/src/model/gp.rs @@ -42,8 +42,9 @@ //! distance `r² = Σᵢ (xᵢ − x′ᵢ)² / ℓᵢ²`. The Matérn kernel with ν = 5/2 by default (eq. 4.17), //! twice differentiable, as Snoek, Larochelle and Adams (2012) advise for Bayesian optimization //! against the squared exponential's infinitely smooth functions (eq. 4.9; Stein, 1999). -//! - **The noise** ([`Noise`]) is learned with the other hyperparameters by default, at least -//! 1e-6 of the values' variance; or fixed, e.g. for interpolation. +//! - **The noise** ([`Noise`]) is none by default: the model interpolates the values, as suits +//! the deterministic functions genoxide optimizes, with the jitter below as the only nugget. +//! For a noisy function, it's learned with the other hyperparameters, or fixed. //! - **The constant mean** `m` is, for given kernel hyperparameters, the one that maximizes the //! marginal likelihood: the generalized least squares estimate `m = 1ᵀK⁻¹y / 1ᵀK⁻¹1` (setting //! the derivative of eq. 2.30 with `y − m` for `y` to zero, as eq. 2.38 models a fixed mean). @@ -58,7 +59,7 @@ //! [`Lbfgsb`](crate::algorithm::Lbfgsb) and the analytic gradient of eq. 5.9, //! `∂/∂θⱼ ln p = ½ tr((ααᵀ − K_y⁻¹) ∂K_y/∂θⱼ)`, `α = K_y⁻¹(y − m)`. The search runs from //! [several starts](GaussianProcessBuilder::starts): the first from fixed values (length scales of -//! 0.5 of each range, `σ_f²` the values' variance, `σ_n²` 1e-4 of it, or the noise's least), the +//! 0.5 of each range, `σ_f²` the values' variance, a learned `σ_n²` 1e-4 of it or its least), the //! others from random points of the box below, drawn from streams derived from the //! [seed](GaussianProcessBuilder::seed), independent of each other. The likelihood's best wins, //! the earlier start on ties, so the fit is the same on any number of threads (with the @@ -125,25 +126,31 @@ impl Kernel { /// The observation noise of a [`GaussianProcess`]: its variance `σ_n²`, as a fraction of the /// values' variance (the model's outputs are standardized). +/// +/// None by default (`Fixed(0.0)`): genoxide's fitness functions are deterministic, and a model +/// that interpolates them resolves the small differences near a minimum that a learned noise +/// smooths over. Measured with [`Bo`](crate::algorithm::Bo) over 20 seeds and 80 evaluations, to +/// f* + 1e-4: Branin reached in 20 runs, the six-hump camel in [−3, 3] × [−2, 2] in 20 and +/// Hartmann 3 in 20, against 14, 17 and 12 with noise learned from a least of 1e-6, which settles +/// above that least, at an absolute standard deviation of about 0.03 on Branin. #[derive(Clone, Copy, Debug, PartialEq)] #[non_exhaustive] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum Noise { - /// Learned with the other hyperparameters, at least `min` (0 < `min` < 1). The default is - /// `min` = 1e-6: for a deterministic function, the model then nearly interpolates its values, - /// while the floor keeps the kernel matrix well conditioned. + /// Learned with the other hyperparameters, at least `min` (0 < `min` < 1), e.g. 1e-6: for a + /// noisy function, whose values the model shouldn't interpolate. Learned { /// The least noise variance, a fraction of the values' variance. min: f64, }, - /// A fixed variance, at least 0: 0 interpolates the values (with the jitter the Cholesky - /// factorization may need). + /// A fixed variance, at least 0: 0 (the default) interpolates the values, with the jitter the + /// Cholesky factorization may need. Fixed(f64), } impl Default for Noise { fn default() -> Self { - Noise::Learned { min: 1e-6 } + Noise::Fixed(0.0) } } @@ -906,8 +913,8 @@ impl Data<'_> { /// A builder for a [`GaussianProcess`], from [`GaussianProcess::builder`]. /// -/// Defaults: the [Matérn 5/2 kernel](Kernel::Matern52), [learned noise](Noise::Learned) of at -/// least 1e-6 of the values' variance, 5 starts of the likelihood's maximization, seed 0. +/// Defaults: the [Matérn 5/2 kernel](Kernel::Matern52), no noise ([`Noise::Fixed`] of 0), 5 starts +/// of the likelihood's maximization, seed 0. #[derive(Clone, Debug)] pub struct GaussianProcessBuilder { real: Real, @@ -925,8 +932,8 @@ impl GaussianProcessBuilder { self } - /// The observation noise: [`Noise::Learned`] with a least variance of 1e-6 of the values' - /// variance by default. + /// The observation noise: none by default ([`Noise::Fixed`] of 0), the model interpolating + /// the values; [`Noise::Learned`] for a noisy function. pub fn noise(mut self, noise: Noise) -> Self { self.noise = noise; self diff --git a/tests/algorithms.rs b/tests/algorithms.rs index 2d53446e..20b7069d 100644 --- a/tests/algorithms.rs +++ b/tests/algorithms.rs @@ -527,19 +527,19 @@ fn portable_runs() { -2.7974966575791904, 1.1930722176383557, -1.3732603836128217, - ], // Bayesian optimization: log-EI, Matérn 5/2 + ], // Bayesian optimization: log-EI, Matérn 5/2 [ - 0.48198052392264934, - 0.055399166715114134, - 0.7923531293628381, - 0.5613182693321956, + 0.7497663253697677, + -0.056696340352855756, + 0.7649350749581538, + 0.7470392106959753, ], // Bayesian optimization: the log transform, PI, the squared exponential [ - 1.1457818657817844, - 1.012510092068803, - 0.8225014439106566, - 0.8156551023913199, + 0.8989959153523532, + 0.8470798321557886, + 0.7794047534046049, + 0.6827343441068123, ], ]; for (run, expected) in runs.iter().zip(expected) { diff --git a/tests/bo.rs b/tests/bo.rs index bdd19d7c..97a9ef08 100644 --- a/tests/bo.rs +++ b/tests/bo.rs @@ -22,7 +22,7 @@ fn evaluations_to_target( #[test] fn branin_in_tens_of_evaluations() { - // three global minima of 0.397887; a median of 30 evaluations over 20 seeds + // three global minima of 0.397887: a median of 30 evaluations over 20 seeds, all within 50 for seed in 1..=3 { let bo = Bo::builder(Branin.representation()) .minimize() @@ -37,13 +37,14 @@ fn branin_in_tens_of_evaluations() { #[test] fn six_hump_camel_in_tens_of_evaluations() { let optimum = SixHumpCamel.optimum().unwrap().value(); - // the usual box for Bayesian optimization, [−3, 3] × [−2, 2]: a median of 49 evaluations + // the usual box for Bayesian optimization, [−3, 3] × [−2, 2]: a median of 50 evaluations + // over 20 seeds, all within 80 let narrow = Real::new([-3.0..=3.0, -2.0..=2.0]).unwrap(); let bo = Bo::builder(narrow).minimize().seed(1).build().unwrap(); assert!(evaluations_to_target(bo, SixHumpCamel, optimum, 70).is_some()); // genoxide's box, [−5, 5]², whose corners reach 6,400: the log transform resolves the - // minima where the standardized values can't (a median of 34 evaluations against 6 of 20 - // runs within 80) + // minima where the standardized values can't (every seed of 20 within 80 evaluations, a + // median of 31, against 4 of 20) let bo = Bo::builder(SixHumpCamel.representation()) .output(Output::Log) .minimize() @@ -55,21 +56,23 @@ fn six_hump_camel_in_tens_of_evaluations() { #[test] fn hartmann_3_in_tens_of_evaluations() { - // 12 of 20 seeds within 80 evaluations, a median of 24; the others stall where the model - // judges x₁ irrelevant, at x₁ = 0, 0.008 above the minimum + // every seed of 20 within 80 evaluations, a median of 26 let optimum = Hartmann3.optimum().unwrap().value(); - let bo = Bo::builder(Hartmann3.representation()) - .minimize() - .seed(3) - .build() - .unwrap(); - assert!(evaluations_to_target(bo, Hartmann3, optimum, 50).is_some()); + for seed in 1..=3 { + let bo = Bo::builder(Hartmann3.representation()) + .minimize() + .seed(seed) + .build() + .unwrap(); + let evaluations = evaluations_to_target(bo, Hartmann3, optimum, 60); + assert!(evaluations.is_some(), "seed {seed}"); + } } #[test] fn goldstein_price_with_the_log_transform() { // values from 3 to 10⁶: no seed of 20 reaches the target within 80 evaluations with - // standardized values, every one with the log transform + // standardized values, 19 with the log transform, a median of 39 let bo = Bo::builder(GoldsteinPrice.representation()) .output(Output::Log) .minimize() diff --git a/tests/checkpoint.rs b/tests/checkpoint.rs index 8f9a603c..f538336b 100644 --- a/tests/checkpoint.rs +++ b/tests/checkpoint.rs @@ -1059,7 +1059,10 @@ fn bayesian_optimization_saved_between_an_ask_and_its_tell_resumes() { let mut resumed: Bo = checkpoint::load(bytes(&bo).as_slice()).unwrap(); let again: Vec> = resumed.ask().iter().map(|x| x.to_vec()).collect(); assert_eq!(again, asked); - let fitness: Vec = asked.iter().map(|x| Fitness::new(branin(&Reals::from(x.clone())))).collect(); + let fitness: Vec = asked + .iter() + .map(|x| Fitness::new(branin(&Reals::from(x.clone())))) + .collect(); bo.tell(&fitness).unwrap(); resumed.tell(&fitness).unwrap(); assert_eq!(bytes(&resumed), bytes(&bo)); @@ -1067,7 +1070,10 @@ fn bayesian_optimization_saved_between_an_ask_and_its_tell_resumes() { let a: Vec> = bo.ask().iter().map(|x| x.to_vec()).collect(); let b: Vec> = resumed.ask().iter().map(|x| x.to_vec()).collect(); assert_eq!(a, b); - let fitness: Vec = a.iter().map(|x| Fitness::new(branin(&Reals::from(x.clone())))).collect(); + let fitness: Vec = a + .iter() + .map(|x| Fitness::new(branin(&Reals::from(x.clone())))) + .collect(); bo.tell(&fitness).unwrap(); resumed.tell(&fitness).unwrap(); } From df9e564e181e283ae7543234e40b2768826903fc Mon Sep 17 00:00:00 2001 From: tachsin Date: Fri, 2 Oct 2026 12:39:56 +0300 Subject: [PATCH 08/13] feat(examples): Bayesian optimization of Branin, its model and acquisition per step Bo's defaults on Branin's function: 6 points of a Latin hypercube, then 24 chosen by log-EI, which end 9.6e-5 above a global minimum; the Gaussian process of the 30 evaluations, its mean minimized by L-BFGS-B from the best point, then evaluated once: 2.1e-5 above it, in 31 evaluations. Over seeds 1 to 20, the polish brings 14 runs within 1e-4 at 30 evaluations, against 4 by the search alone. Rust and Python print the same rows and write the same trace, whose frames hold the model's mean and the acquisition on a 25 x 25 grid, for a new plot kind of the site's player, `surrogate`, and a new category, `bayesian`. --- docs/hooks/examples.py | 1 + examples/README.md | 3 +- examples/bayesian_optimization/README.md | 89 +++++++++ examples/bayesian_optimization/main.py | 77 ++++++++ examples/bayesian_optimization/main.rs | 138 +++++++++++++ examples/bayesian_optimization/output.txt | 36 ++++ examples/bayesian_optimization/trace.json | 28 +++ examples/bayesian_optimization/trace.py | 139 +++++++++++++ examples/bayesian_optimization/trace.rs | 179 +++++++++++++++++ .../projects/genoxide/player/PlayerView.jsx | 2 + .../genoxide/player/plots/SurrogatePlot.jsx | 186 ++++++++++++++++++ 11 files changed, 877 insertions(+), 1 deletion(-) create mode 100644 examples/bayesian_optimization/README.md create mode 100644 examples/bayesian_optimization/main.py create mode 100644 examples/bayesian_optimization/main.rs create mode 100644 examples/bayesian_optimization/output.txt create mode 100644 examples/bayesian_optimization/trace.json create mode 100644 examples/bayesian_optimization/trace.py create mode 100644 examples/bayesian_optimization/trace.rs create mode 100644 site/components/projects/genoxide/player/plots/SurrogatePlot.jsx diff --git a/docs/hooks/examples.py b/docs/hooks/examples.py index c3ca53c1..16b077fa 100644 --- a/docs/hooks/examples.py +++ b/docs/hooks/examples.py @@ -30,6 +30,7 @@ "integer", "continuous", "local", + "bayesian", "multi-objective", "constrained", "neuroevolution", diff --git a/examples/README.md b/examples/README.md index 11bbfebe..fc16fc62 100644 --- a/examples/README.md +++ b/examples/README.md @@ -211,6 +211,7 @@ python examples/tsp_berlin52/main.py | [Adam with a learning-rate schedule](adam/) | local | Rust, Python | [tachsin.gr](https://tachsin.gr/projects/genoxide/examples/adam) | | [MMA on a million variables](mma/) | local | Rust, Python | [tachsin.gr](https://tachsin.gr/projects/genoxide/examples/mma) | | [Continuation by Gaussian smoothing](continuation/) | local | Rust, Python | [tachsin.gr](https://tachsin.gr/projects/genoxide/examples/continuation) | +| [Bayesian optimization of Branin](bayesian_optimization/) | bayesian | Rust, Python | [tachsin.gr](https://tachsin.gr/projects/genoxide/examples/bayesian-optimization) | The GPU example is a crate of its own, with wgpu as a dependency: `cargo run --release --manifest-path examples/gpu/Cargo.toml`. @@ -222,4 +223,4 @@ feature it needs, which its README says) and a `README.md` with the front matter table above. Cargo finds `main.rs` as the example ``, CI runs every `main.rs` and `main.py`, and the docs site makes a page for it, whose build fails if the table lacks the example. The categories are binary, permutation, integer, continuous, multi-objective, constrained, -neuroevolution and engine. +neuroevolution, genetic programming, engine, local and bayesian. diff --git a/examples/bayesian_optimization/README.md b/examples/bayesian_optimization/README.md new file mode 100644 index 00000000..3f655ed2 --- /dev/null +++ b/examples/bayesian_optimization/README.md @@ -0,0 +1,89 @@ +--- +title: Bayesian optimization of Branin +category: bayesian +summary: Find a global minimum of Branin's function in 31 evaluations, with a Gaussian process that chooses each point by the log expected improvement and a final L-BFGS-B polish of the model's mean. +reference: "Jones, D. R., Schonlau, M. and Welch, W. J. (1998). Efficient global optimization of expensive black-box functions. Journal of Global Optimization 13(4): 455-492." +reference_url: "https://doi.org/10.1023/A:1008306431147" +optimum: "5 / (4π) ≈ 0.397887 (at three points)" +languages: [rust, python] +order: 290 +--- + +# Bayesian optimization of Branin + +## The problem + +Branin's function, in the form Dixon and Szegö (1978) give it, is + +```text +f(x₁, x₂) = (x₂ − 5.1 x₁² / (4π²) + 5 x₁ / π − 6)² + 10 (1 − 1 / (8π)) cos x₁ + 10 +``` + +on x₁ in [−5, 10] and x₂ in [0, 15]. Its three global minima, all worth 5 / (4π) ≈ 0.397887, are at +(−π, 12.275), (π, 2.275) and (3π, 2.475). It's one of the problems on which Jones, Schonlau and +Welch (1998) introduced efficient global optimization, the method this example runs: Bayesian +optimization with the expected improvement. + +Here the function stands for an expensive one, a simulation that runs for minutes or an +experiment, where every evaluation counts. The question isn't how fast the search runs but how +few evaluations it needs. + +## What makes it hard + +Nothing, for a method with thousands of evaluations to spend: the [Branin](../branin/) example +finds all three minima with 30 hill climbers of 10,001 evaluations each. With tens of +evaluations, every point has to be chosen with care: where the function is probably low, and +where too little is known to tell. The values range from 0.4 to over 300, while the minima's +basins are narrow valleys. + +## Representation + +A `Real` genome of 2 genes, x₁ in [−5, 10] and x₂ in [0, 15]: genoxide's `problems::Branin`, +whose box and minima are those above, evaluated in Rust in both languages. + +## Algorithm + +`Bo`, genoxide's Bayesian optimization, with its defaults: + +- **The initial design**: 2(n + 1) = 6 points of a Latin hypercube, one in each sixth of each + gene's range. +- **The model**: a Gaussian process with a constant mean and Matérn's 5/2 kernel, a length scale + per gene, fitted to every evaluation so far by maximizing its marginal likelihood (Rasmussen and + Williams 2006), without noise, since the function is deterministic: the model passes through + every value. +- **The next point**: the maximum of the log expected improvement (Ament et al. 2023), the + logarithm of how much a point is expected to beat the best value under the model, computed + so that it keeps a gradient where the expected improvement itself underflows. It's evaluated + at 1,000 random points, then maximized by L-BFGS-B from the best 10 of them and from the best + point so far. + +After 30 evaluations, a Gaussian process of all of them is fitted, and its posterior mean, +smooth and cheap, is minimized by L-BFGS-B with its gradient from the best point found. The +function is evaluated once at the result, the 31st evaluation. + +## Output + +The first lines give the problem and the method. Then a row per evaluation: its number, the point, +the function's value and its distance above the global minimum. Generation 0 is the six points of +the design; each later row is a point the model chose. The last lines give the polish: the +point where the model's mean is lowest, the mean there, the function's value, its distance above +the nearest minimum, and the best of the 31 evaluations. In Python, `run` evaluates the function +in Rust and the model is genoxide's, so both versions print the same rows. + +[The project page](https://tachsin.gr/projects/genoxide/examples/bayesian-optimization) plays the +run back: at each step, the model's mean over the box with the points so far, the log expected +improvement that chose the next point, and a curve of the best value's distance above the minimum. + +## Good results + +A good result is within 1e-4 of a global minimum, 0.397887, in tens of evaluations. The first +point the model chose, the 7th evaluation, is already 2.5 above it; the 16th is 0.043 above it, +near (π, 2.275), and from there the search visits all three basins, the 24th within 1.3e-3 of +(−π, 12.275), the 29th within 9.6e-5 of (3π, 2.475). The model of the 30 evaluations has its +lowest mean at (9.422974, 2.475867), 0.397917, and the function there is 0.397909, 2.1e-5 above +the minimum: 31 evaluations in all. + +Over seeds 1 to 20, the search reaches 1e-3 in a median of 30 evaluations, every seed within 80. +After 30 evaluations, 4 searches are within 1e-4, and 14 with the polish; after 35, 18, and 20 +with the polish. The polish costs one evaluation: the model, which passes through every value, +is most precise near the best points, where the search has evaluated most. diff --git a/examples/bayesian_optimization/main.py b/examples/bayesian_optimization/main.py new file mode 100644 index 00000000..d2aee778 --- /dev/null +++ b/examples/bayesian_optimization/main.py @@ -0,0 +1,77 @@ +"""Bayesian optimization of Branin's function: 30 evaluations chosen by a Gaussian process and the +log expected improvement, then the model's mean minimized by L-BFGS-B and evaluated once. + +The search comes within 1e-4 of one of the three global minima, and the polish of the model, for +one evaluation more, closes most of what is left. + +With ``GENOXIDE_TRACE=``, it also writes a trace of its run for the plot on the example's +page, with trace.py. + + python examples/bayesian_optimization/main.py +""" + +import numpy as np + +import genoxide as gx + +from trace import Trace + +# the evaluations of the search: the initial design, then the points the model chooses +EVALUATIONS = 30 + + +def scientific(value): + """Two significant digits, e.g. 1.2e-7.""" + mantissa, exponent = f"{value:.1e}".split("e") + return f"{mantissa}e{int(exponent)}" + + +problem = gx.problems.Branin() +minima = problem.optimum.solutions +minimum = problem.optimum.value +bo = gx.Bo(problem.genome, objective="minimize", seed=1) +print(f"Branin's function in [-5, 10] x [0, 15]: three global minima of {minimum:.6f}") +# the initial design's default size, 2(n + 1) for n = 2 genes +print("6 points of a Latin hypercube, then a point per step by log-EI on a Gaussian process") +print("evaluation x1 x2 f f - f*") +# with GENOXIDE_TRACE=, a trace of the run for the plot on the example's page +trace = Trace(minima, minimum) +evaluated = {"points": None, "values": None} + + +def on_generation(progress): + # the points evaluated in this generation + printed = 0 if evaluated["points"] is None else len(evaluated["points"]) + for index in range(printed, len(progress.population)): + x, value = progress.population[index], float(progress.scores[index]) + print( + f"{index + 1:>10} {x[0]:>10.6f} {x[1]:>10.6f} {value:>12.6f} " + f"{scientific(value - minimum):>10}" + ) + evaluated["points"], evaluated["values"] = progress.population, progress.scores + + +result = bo.run(problem, evaluations=EVALUATIONS, on_generation=on_generation, control=trace.record) + +# a Gaussian process of every evaluation, its mean minimized from the best point +points, values = evaluated["points"], evaluated["values"] +model = gx.model.gp.GaussianProcess.fit(problem.genome, points, values) +lbfgsb = gx.Lbfgsb(problem.genome, initial_genome=result.best_genome, objective="minimize", seed=1) +polished = lbfgsb.run( + lambda x: model.predict(x)[0][0], + gradient=lambda x: model.predict_with_gradient(x)[2], + evaluations=1_000, +) +x = polished.best_genome +# one evaluation of the function there +value = float(problem(x)) +nearest = minima[int(np.argmin(np.linalg.norm(minima - x, axis=1)))] +print(f"the model of the {len(points)} evaluations, its mean minimized by L-BFGS-B from the best point:") +print( + f"({x[0]:.6f}, {x[1]:.6f}): predicted {model.predict(x)[0][0]:.6f}, evaluated {value:.6f}, " + f"{scientific(value - minimum)} above the minimum at ({nearest[0]:.6f}, {nearest[1]:.6f})" +) +best = min(result.best_fitness, value) +print(f"{len(points) + 1} evaluations: the best {scientific(best - minimum)} above the global minimum") +assert best - minimum <= 1e-4 +trace.write(x, value) diff --git a/examples/bayesian_optimization/main.rs b/examples/bayesian_optimization/main.rs new file mode 100644 index 00000000..55f78c8f --- /dev/null +++ b/examples/bayesian_optimization/main.rs @@ -0,0 +1,138 @@ +//! Bayesian optimization of Branin's function: 30 evaluations chosen by a Gaussian process and the +//! log expected improvement, then the model's mean minimized by L-BFGS-B and evaluated once. +//! +//! The search comes within 1e-4 of one of the three global minima, and the polish of the model, +//! for one evaluation more, closes most of what is left. +//! +//! With `GENOXIDE_TRACE=`, it also writes a trace of its run for the plot on the example's +//! page, with `trace.rs`. +//! +//! ```text +//! cargo run --release --example bayesian_optimization +//! ``` + +mod trace; + +use genoxide::model::gp::GaussianProcess; +use genoxide::prelude::*; +use genoxide::problems::{Branin, Problem}; +use std::cell::RefCell; + +// the evaluations of the search: the initial design, then the points the model chooses +const EVALUATIONS: u64 = 30; + +fn main() -> Result<()> { + let problem = Branin; + let optimum = problem.optimum().expect("known"); + let minimum = optimum.value(); + let bo = Bo::builder(problem.representation()) + .minimize() + .seed(1) + .build()?; + println!("Branin's function in [-5, 10] x [0, 15]: three global minima of {minimum:.6}"); + println!( + "{} points of a Latin hypercube, then a point per step by log-EI on a Gaussian process", + bo.initial_points() + ); + println!("evaluation x1 x2 f f - f*"); + // with GENOXIDE_TRACE=, a trace of the run for the plot on the example's page + let trace = RefCell::new(trace::Trace::from_env()); + let mut printed = 0; + let mut engine = Engine::new(bo, problem) + .stop_when(Stop::evaluations(EVALUATIONS)) + .on_generation(|snapshot| { + // the points evaluated in this generation + let population = snapshot.population().as_slice(); + for (index, individual) in population.iter().enumerate().skip(printed) { + let x = individual.genome(); + let value = individual + .fitness() + .and_then(Fitness::score) + .expect("valid"); + println!( + "{:>10} {:>10.6} {:>10.6} {:>12.6} {:>10}", + index + 1, + x[0], + x[1], + value, + scientific(value - minimum) + ); + } + printed = population.len(); + }) + .control(|bo, progress| { + trace.borrow_mut().record(bo, progress); + Ok(()) + }); + let outcome = engine.run()?; + let best = outcome.best_genome().clone(); + let best_value = outcome.best_fitness().score().expect("valid"); + + // a Gaussian process of every evaluation, its mean minimized from the best point + let evaluated = engine.algorithm().population(); + let points: Vec = evaluated.iter().map(|x| x.genome().clone()).collect(); + let values: Vec = evaluated + .iter() + .map(|x| x.fitness().and_then(Fitness::score).expect("valid")) + .collect(); + // the engine borrows the trace in its control + drop(engine); + let model = GaussianProcess::builder(problem.representation()).fit(&points, &values)?; + let mean = Differentiable(|x: &Reals, gradient: &mut [f64]| { + let mut variance_gradient = [0.0; 2]; + let prediction = model.predict_with_gradient(x, gradient, &mut variance_gradient); + prediction.mean() + }); + let lbfgsb = Lbfgsb::builder(problem.representation()) + .initial_genome(best) + .minimize() + .seed(1) + .build()?; + let polished = Engine::new(lbfgsb, mean) + .stop_when(Stop::evaluations(1_000)) + .run()?; + let x = polished.best_genome(); + // one evaluation of the function there + let value = problem.evaluate(x); + let nearest = optimum + .solutions() + .iter() + .min_by(|a, b| distance(a, x).total_cmp(&distance(b, x))) + .expect("three minima"); + println!( + "the model of the {} evaluations, its mean minimized by L-BFGS-B from the best point:", + points.len() + ); + println!( + "({:.6}, {:.6}): predicted {:.6}, evaluated {value:.6}, {} above the minimum at \ + ({:.6}, {:.6})", + x[0], + x[1], + model.predict(x).mean(), + scientific(value - minimum), + nearest[0], + nearest[1] + ); + let best_value = best_value.min(value); + println!( + "{} evaluations: the best {} above the global minimum", + points.len() + 1, + scientific(best_value - minimum) + ); + assert!(best_value - minimum <= 1e-4); + trace.into_inner().write(x, value); + Ok(()) +} + +fn distance(a: &[f64], b: &[f64]) -> f64 { + a.iter() + .zip(b) + .map(|(x, y)| (x - y) * (x - y)) + .sum::() + .sqrt() +} + +// two significant digits, e.g. 1.2e-7 +fn scientific(value: f64) -> String { + format!("{value:.1e}") +} diff --git a/examples/bayesian_optimization/output.txt b/examples/bayesian_optimization/output.txt new file mode 100644 index 00000000..58d7b7c5 --- /dev/null +++ b/examples/bayesian_optimization/output.txt @@ -0,0 +1,36 @@ +Branin's function in [-5, 10] x [0, 15]: three global minima of 0.397887 +6 points of a Latin hypercube, then a point per step by log-EI on a Gaussian process +evaluation x1 x2 f f - f* + 1 -2.094023 11.483177 7.711008 7.3e0 + 2 9.566534 8.178619 31.646772 3.1e1 + 3 1.779636 0.917823 15.079196 1.5e1 + 4 3.116619 3.157308 1.145222 7.5e-1 + 5 -4.412273 6.407455 90.516029 9.0e1 + 6 5.122378 13.384503 161.386238 1.6e2 + 7 3.802460 1.091098 2.945049 2.5e0 + 8 5.754346 1.947420 18.976188 1.9e1 + 9 -4.083638 13.352664 6.045107 5.6e0 + 10 -2.253838 14.243188 19.938494 2.0e1 + 11 4.727562 5.297559 25.625619 2.5e1 + 12 10.000000 3.801580 2.580940 2.2e0 + 13 10.000000 0.000000 10.960889 1.1e1 + 14 0.890363 6.322041 18.719724 1.8e1 + 15 -5.000000 15.000000 17.508300 1.7e1 + 16 3.235854 2.197541 0.440540 4.3e-2 + 17 9.097758 2.736199 1.180535 7.8e-1 + 18 9.790067 2.374804 1.212532 8.1e-1 + 19 8.300793 0.000000 8.707440 8.3e0 + 20 2.863389 2.491949 0.767187 3.7e-1 + 21 -3.154554 12.141734 0.425733 2.8e-2 + 22 -3.156264 12.562523 0.462544 6.5e-2 + 23 9.492983 2.838771 0.513627 1.2e-1 + 24 -3.153503 12.328721 0.399197 1.3e-3 + 25 9.363618 2.249384 0.446294 4.8e-2 + 26 -3.001106 11.914866 0.493115 9.5e-2 + 27 3.134020 2.378650 0.407715 9.8e-3 + 28 3.208078 2.505644 0.498582 1.0e-1 + 29 9.424509 2.484571 0.397984 9.6e-5 + 30 -3.833119 15.000000 3.606387 3.2e0 +the model of the 30 evaluations, its mean minimized by L-BFGS-B from the best point: +(9.422974, 2.475867): predicted 0.397917, evaluated 0.397909, 2.1e-5 above the minimum at (9.424778, 2.475000) +31 evaluations: the best 2.1e-5 above the global minimum diff --git a/examples/bayesian_optimization/trace.json b/examples/bayesian_optimization/trace.json new file mode 100644 index 00000000..48f10991 --- /dev/null +++ b/examples/bayesian_optimization/trace.json @@ -0,0 +1,28 @@ +{"example":"bayesian_optimization","format":1,"log_y":true,"objective":"minimize","optimum":0.0,"plot":"surrogate","problem":{"bounds":[[-5.0,10.0],[0.0,15.0]],"function":"branin","grid":25,"minima":[[-3.14159,12.275],[3.14159,2.275],[9.42478,2.475]]},"x_label":"evaluations","y_label":"error to the global minimum","frames":[ +{"best":0.747335,"evaluations":6,"generation":0,"state":{"best":[3.11662,3.15731],"newest":[5.12238,13.3845],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845]]}}, +{"best":0.747335,"evaluations":7,"generation":1,"state":{"acquisition":[[966,966,967,969,971,974,976,977,975,964,937,929,974,993,997,996,992,987,984,980,978,977,976,975,975],[963,963,964,967,970,973,976,978,976,963,914,802,971,995,999,998,994,989,985,981,979,978,977,976,975],[959,958,960,963,968,972,976,978,978,968,934,842,973,996,1000,999,995,990,986,982,980,978,977,977,976],[952,952,954,959,965,970,976,979,981,977,967,961,980,992,997,998,995,991,987,983,981,979,978,978,977],[943,941,945,953,961,969,975,980,984,985,986,987,984,976,990,996,995,991,987,983,981,980,979,978,978],[928,925,932,944,956,966,974,981,986,990,994,996,986,867,980,993,993,990,987,984,982,980,980,979,979],[905,899,911,932,950,964,973,981,987,993,997,999,990,962,979,991,992,989,986,983,982,981,980,979,979],[867,850,877,916,944,961,973,981,987,993,998,999,993,982,984,989,990,988,985,983,981,980,979,979,979],[798,740,820,897,938,960,972,981,987,992,996,997,994,988,987,988,987,985,983,981,980,979,978,977,977],[670,351,724,879,934,959,973,981,987,991,994,995,993,989,987,985,984,982,980,979,978,977,974,971,972],[515,0,626,871,934,961,974,982,986,989,991,992,990,986,983,981,979,977,976,976,975,973,966,959,962],[598,0,677,880,939,964,977,983,986,988,989,988,985,982,978,974,972,971,971,972,971,967,952,931,941],[764,650,796,901,947,969,980,985,987,987,986,983,980,975,970,966,963,962,964,966,966,959,932,848,895],[857,833,874,925,956,974,983,987,987,986,983,979,974,967,960,954,951,951,954,959,960,953,916,459,834],[907,901,919,944,965,978,986,989,988,985,980,974,967,958,948,939,934,936,942,949,954,949,920,803,879],[936,936,946,958,970,980,987,990,988,984,978,970,959,946,932,918,911,914,924,937,946,948,935,911,929],[954,956,962,967,969,976,987,991,989,983,975,965,951,933,910,888,876,881,900,922,938,946,946,943,952],[965,969,973,972,960,962,984,991,988,982,972,959,942,917,882,843,818,830,866,902,928,944,952,956,963],[973,977,980,977,943,919,981,990,988,981,970,954,933,899,846,772,714,743,818,879,918,941,954,962,968],[977,982,985,982,956,940,983,990,987,979,967,950,924,880,800,651,482,574,752,854,907,937,953,963,970],[980,985,988,987,978,976,988,991,987,978,965,946,917,864,752,450,0,199,671,830,898,933,952,963,970],[982,986,990,991,989,989,991,991,985,976,963,944,912,854,718,218,0,0,608,815,893,931,951,963,969],[982,986,990,993,993,993,993,990,984,975,962,943,912,854,720,244,0,0,613,815,892,930,950,962,968],[982,986,989,992,993,993,992,988,982,974,962,944,915,863,755,482,0,277,680,830,897,931,950,961,968],[981,984,988,990,992,991,990,986,981,973,962,945,920,878,802,668,527,602,758,853,905,934,951,961,967]],"best":[3.11662,3.15731],"mean":[[810,809,807,803,796,787,774,755,729,692,646,604,585,600,637,679,715,741,759,771,779,784,786,788,789],[815,815,812,807,799,788,773,752,721,676,615,549,511,530,588,650,698,731,754,768,776,781,784,786,787],[821,821,818,812,803,791,774,751,716,665,589,491,410,430,527,617,681,722,748,764,773,779,781,783,784],[829,829,826,818,808,794,775,751,714,659,574,443,273,274,454,586,666,715,744,761,771,775,778,779,781],[838,839,834,826,813,797,778,752,716,661,574,426,133,16,392,565,658,711,741,759,768,772,774,775,776],[849,849,844,833,819,801,781,755,720,668,587,451,182,0,386,564,658,711,741,757,765,768,768,769,770],[859,861,854,841,824,805,783,758,726,680,611,504,349,285,449,585,667,715,743,757,762,763,762,761,763],[870,871,864,848,828,807,785,761,732,693,639,565,481,458,532,621,685,724,747,758,760,758,754,751,753],[879,881,871,853,831,807,785,763,738,707,667,619,575,567,606,662,707,737,754,760,759,752,744,739,741],[885,887,876,855,829,805,782,762,742,720,693,664,642,641,666,701,732,753,763,765,758,747,734,725,726],[886,888,875,851,824,798,776,759,745,730,714,699,691,695,713,736,757,770,775,771,760,742,723,710,710],[882,882,868,842,812,786,766,753,745,738,732,728,728,736,751,768,782,790,789,781,764,741,714,695,695],[872,870,854,826,794,767,750,743,742,744,746,750,757,769,783,797,807,810,806,793,772,743,710,685,684],[858,852,833,802,767,740,727,728,737,747,758,769,781,796,811,824,832,832,824,808,783,751,713,684,681],[841,831,807,771,731,702,696,709,729,749,767,784,802,820,837,850,857,855,845,825,798,763,725,695,689],[823,808,777,732,682,651,656,686,720,750,775,797,819,841,861,876,882,879,867,845,815,780,743,714,706],[805,784,745,687,619,582,608,662,711,750,782,809,835,861,884,900,908,904,889,865,834,798,763,736,725],[789,762,714,640,545,498,557,640,705,752,788,820,850,879,905,924,933,928,911,884,851,816,782,757,745],[775,744,690,601,476,420,521,628,702,754,794,829,862,894,924,946,956,951,932,902,868,832,799,775,762],[765,732,676,584,456,407,519,629,705,759,800,836,872,907,940,965,976,971,949,917,881,845,814,790,776],[760,728,675,594,497,468,551,644,714,764,805,842,879,916,951,978,992,985,962,929,891,855,824,801,787],[758,729,684,622,559,545,598,667,725,771,810,847,884,921,957,986,1000,993,969,935,897,862,832,809,795],[760,735,699,655,617,610,643,692,738,778,814,849,885,922,957,986,1000,993,969,936,899,865,836,815,801],[763,743,716,686,662,659,681,715,750,784,816,848,883,918,952,978,991,985,963,932,897,865,838,818,805],[767,752,732,712,697,696,710,734,762,789,817,846,878,910,941,965,977,971,952,923,892,862,838,819,807]],"newest":[3.80246,1.0911],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911]]}}, +{"best":0.747335,"evaluations":8,"generation":2,"state":{"acquisition":[[934,934,934,934,934,934,934,934,934,934,934,934,934,934,934,934,934,934,934,934,934,934,934,934,934],[786,786,785,785,785,785,785,785,785,785,785,785,785,786,786,786,786,786,786,786,786,786,786,786,786],[961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961,961],[1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000],[994,994,994,994,994,994,994,994,994,994,994,994,994,994,994,994,994,994,994,994,994,994,994,994,994],[896,896,896,896,896,895,895,895,895,895,895,895,895,895,895,895,895,895,895,895,895,895,896,896,896],[973,973,973,973,973,973,973,973,973,973,973,973,973,973,973,973,973,973,973,973,973,973,973,973,973],[968,968,968,968,968,968,968,968,968,968,968,968,968,968,968,968,968,968,968,968,968,968,968,968,968],[949,949,949,949,949,949,949,949,949,949,949,949,949,949,949,949,949,949,949,949,949,949,949,949,949],[896,896,896,896,896,896,896,896,896,896,896,896,896,896,896,896,896,896,896,896,896,896,896,896,896],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[796,796,796,796,796,796,796,796,796,796,796,796,796,796,796,796,796,796,796,796,796,796,796,796,796],[915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915,915],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[946,946,946,946,946,946,946,946,946,946,946,946,946,946,946,946,946,946,946,946,946,946,946,946,946],[967,967,967,967,967,967,967,967,967,967,967,967,967,967,967,967,967,967,967,967,967,967,967,967,967],[972,972,972,972,972,972,972,972,972,972,972,972,972,972,972,972,972,972,972,972,972,972,972,972,972],[979,979,979,979,979,979,979,979,979,979,979,979,979,979,979,979,979,979,979,979,979,979,979,979,979],[951,951,951,951,951,951,951,951,951,951,951,951,951,951,951,951,951,951,951,951,951,951,951,951,951],[930,930,930,930,930,930,930,930,930,930,930,930,930,930,930,930,930,930,930,930,930,930,930,930,930],[858,858,858,858,858,858,858,858,858,858,858,858,858,858,858,858,858,858,858,858,858,858,858,858,858],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[310,310,310,310,310,310,310,310,310,310,310,310,310,310,310,310,310,310,310,310,310,310,310,310,310],[862,862,862,862,862,862,862,862,862,862,862,862,862,862,862,862,862,862,862,862,862,862,862,862,862],[932,932,932,932,932,932,932,932,932,932,932,932,932,932,932,932,932,932,932,932,932,932,932,932,932]],"best":[3.11662,3.15731],"mean":[[799,799,799,799,799,799,799,799,799,799,799,799,799,799,799,799,799,799,799,799,799,799,799,799,799],[730,730,730,730,730,730,730,730,730,730,730,730,730,730,730,730,730,730,730,730,730,730,730,730,730],[1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[261,261,261,261,261,261,261,261,261,261,261,261,261,261,261,261,261,261,261,261,261,261,261,261,261],[441,441,441,441,441,441,441,441,441,441,441,441,441,441,441,441,441,441,441,441,441,441,441,441,441],[369,369,369,369,369,369,369,369,369,369,369,369,369,369,369,369,369,369,369,369,369,369,369,369,369],[630,630,630,630,630,630,630,630,630,630,630,630,630,630,630,630,630,630,630,630,630,630,630,630,630],[754,754,754,754,754,754,754,754,754,754,754,754,754,754,754,754,754,754,754,754,754,754,754,754,754],[814,814,814,814,814,814,814,814,814,814,814,814,814,814,814,814,814,814,814,814,814,814,814,814,814],[865,865,865,865,865,865,865,865,865,865,865,865,865,865,865,865,865,865,865,865,865,865,865,865,865],[903,903,903,903,903,903,903,903,903,903,903,903,903,903,903,903,903,903,903,903,903,903,903,903,903],[878,878,878,878,878,878,878,878,878,878,878,878,878,878,878,878,878,878,878,878,878,878,878,878,878],[800,800,800,800,800,800,800,800,800,800,800,800,800,800,800,800,800,800,800,800,800,800,800,800,800],[720,720,720,720,720,720,720,720,720,720,720,720,720,720,720,720,720,720,720,720,720,720,720,720,720],[728,728,728,728,728,728,728,728,728,728,728,728,728,728,728,728,728,728,728,728,728,728,728,728,728],[755,755,755,755,755,755,755,755,755,755,755,755,755,755,755,755,755,755,755,755,755,755,755,755,755],[745,745,745,745,745,745,745,745,745,745,745,745,745,745,745,745,745,745,745,745,745,745,745,745,745],[669,669,669,669,669,669,669,669,669,669,669,669,669,669,669,669,669,669,669,669,669,669,669,669,669],[498,498,498,498,498,498,498,498,498,498,498,498,498,498,498,498,498,498,498,498,498,498,498,498,498],[691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691,691],[895,895,895,895,895,895,895,895,895,895,895,895,895,895,895,895,895,895,895,895,895,895,895,895,895],[1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000,1000],[998,998,998,998,998,998,998,998,998,998,998,998,998,998,998,998,998,998,998,998,998,998,998,998,998],[923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923,923],[854,854,854,854,854,854,854,854,854,854,854,854,854,854,854,854,854,854,854,854,854,854,854,854,854]],"newest":[5.75435,1.94742],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742]]}}, +{"best":0.747335,"evaluations":9,"generation":3,"state":{"acquisition":[[962,963,963,963,962,959,954,946,931,910,892,906,936,952,958,961,963,962,960,960,962,966,969,972,973],[958,958,958,959,958,956,951,941,921,878,756,669,885,922,916,929,937,935,933,939,948,957,964,969,973],[951,952,952,953,954,953,950,942,926,894,817,767,914,942,896,897,886,854,865,902,931,949,960,968,972],[942,942,944,946,949,950,951,948,942,933,927,938,963,976,973,951,872,0,647,862,918,945,959,968,973],[928,928,931,936,941,947,951,954,955,956,958,962,967,974,978,968,930,837,814,884,926,948,962,970,975],[907,907,911,920,930,940,949,957,963,967,969,966,952,869,964,971,960,938,924,931,945,958,967,973,977],[872,871,880,895,914,931,945,957,965,971,973,969,957,946,966,976,975,967,961,959,963,968,973,977,980],[811,805,824,857,891,918,939,955,965,972,976,975,971,971,977,983,983,980,977,974,974,976,978,980,982],[685,655,712,797,861,904,933,952,965,973,978,979,980,981,984,986,987,986,983,981,980,979,979,980,981],[352,105,444,710,832,894,929,951,965,974,979,981,983,984,986,987,987,986,985,983,980,978,976,976,977],[0,0,0,644,819,891,929,952,966,974,979,982,983,983,984,984,984,983,982,979,975,971,966,963,965],[81,0,223,690,836,900,935,955,967,974,978,980,980,980,979,979,978,977,975,971,965,955,942,932,938],[632,555,658,791,871,917,944,960,970,975,976,976,975,973,971,970,968,967,964,959,949,930,888,823,858],[815,800,823,867,907,936,955,966,972,974,974,971,968,964,960,957,954,952,949,943,930,900,806,0,605],[894,890,897,914,934,951,963,971,973,973,969,964,957,950,943,938,934,931,928,922,910,883,817,685,779],[936,935,937,942,951,961,969,973,973,969,962,953,942,931,920,911,904,901,899,896,891,879,860,852,880],[961,961,959,958,958,962,968,971,970,963,953,939,922,904,886,870,859,855,857,862,868,873,879,891,911],[977,977,974,965,950,947,959,965,963,955,940,920,895,865,834,805,786,783,794,815,838,860,881,902,921],[987,987,983,969,919,866,942,957,955,944,925,897,859,810,751,690,650,651,691,745,797,840,874,901,923],[994,994,991,980,947,913,945,954,949,934,909,872,816,734,613,458,338,367,507,646,746,815,862,895,919],[997,998,997,991,980,968,964,959,947,927,896,848,773,642,394,0,0,0,174,518,692,790,849,888,915],[998,1000,1000,997,991,983,974,963,948,924,889,834,743,571,158,0,0,0,0,415,656,774,841,883,911],[998,1000,1000,998,994,987,978,966,949,924,888,833,743,572,173,0,0,0,0,425,658,774,840,882,910],[996,998,998,997,993,987,979,967,950,927,893,844,768,639,404,0,0,0,233,535,697,790,847,885,911],[994,995,995,994,991,985,978,967,951,930,902,862,804,721,603,459,360,397,528,656,751,816,861,892,915]],"best":[3.11662,3.15731],"mean":[[817,814,809,802,793,782,767,748,724,694,657,613,566,527,518,545,591,639,679,710,733,750,763,772,778],[823,819,814,807,796,783,765,743,714,677,628,567,493,425,412,468,546,613,664,700,726,745,758,768,775],[830,827,821,812,801,785,765,739,706,661,601,518,405,279,262,385,506,593,653,693,721,740,754,764,771],[839,836,829,820,807,790,768,739,701,650,580,477,318,77,84,327,484,582,645,687,715,735,749,758,766],[849,846,839,829,816,797,774,743,702,648,573,462,286,0,65,328,483,579,641,682,710,729,742,752,759],[860,857,850,840,825,806,781,750,709,656,584,484,347,207,240,385,503,584,640,678,704,721,734,743,751],[871,868,861,851,835,815,790,759,720,672,610,533,447,384,395,464,538,599,644,676,698,713,724,733,740],[881,879,872,860,844,824,799,770,734,692,643,589,537,504,507,539,581,621,653,676,693,705,714,721,728],[889,887,880,868,851,831,807,779,748,713,676,638,607,588,588,602,624,647,666,681,690,697,702,707,713],[894,891,884,871,854,834,810,785,758,731,704,679,660,649,648,654,665,675,684,689,691,691,691,693,698],[892,890,882,868,851,831,809,787,765,745,726,711,701,696,695,698,702,705,705,702,696,689,683,680,683],[884,881,872,858,841,821,802,783,767,753,743,737,734,733,734,736,736,734,729,720,708,694,681,673,672],[869,864,853,839,822,804,787,773,763,758,756,757,761,764,768,770,769,765,757,744,727,708,689,675,671],[846,838,826,810,793,776,764,757,755,759,766,775,784,792,799,802,802,797,787,773,754,732,709,691,683],[818,806,789,771,752,738,731,733,743,757,774,791,806,819,828,833,833,829,820,805,786,763,740,720,709],[784,766,743,719,698,687,689,705,728,756,782,807,827,844,856,863,865,861,853,839,820,798,776,755,741],[748,720,688,654,627,621,639,674,715,756,792,823,848,869,883,892,895,893,885,871,854,833,811,790,774],[711,673,627,577,541,543,586,647,707,760,803,840,869,892,909,920,924,922,915,902,884,864,842,822,804],[681,633,572,504,457,476,551,635,708,769,817,857,889,914,933,945,950,949,941,928,911,891,869,848,829],[662,610,544,472,430,466,554,644,720,782,832,873,906,933,953,966,972,971,963,950,932,912,890,869,849],[658,610,554,501,480,517,592,670,740,798,846,887,920,947,968,982,989,988,980,966,948,928,906,885,864],[668,630,591,560,555,586,641,703,762,813,858,897,929,956,977,991,998,997,989,975,958,937,916,895,875],[685,658,634,618,621,645,686,734,782,827,867,903,933,959,979,993,1000,999,991,978,961,941,921,900,881],[705,687,673,666,672,691,722,759,799,837,872,904,932,956,975,988,994,994,987,974,958,940,921,901,883],[725,713,705,703,710,726,750,779,811,843,873,902,927,948,965,977,983,982,976,965,951,934,916,898,881]],"newest":[-4.08364,13.3527],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527]]}}, +{"best":0.747335,"evaluations":10,"generation":4,"state":{"acquisition":[[966,966,966,965,964,961,955,945,928,902,878,896,935,955,960,964,965,963,961,962,966,972,977,981,984],[959,959,959,959,958,956,950,938,915,865,716,588,878,923,915,929,934,930,929,938,951,962,972,979,983],[950,950,950,951,952,951,947,939,920,882,785,703,909,945,891,892,872,829,848,896,932,954,968,977,983],[937,937,938,941,944,947,947,944,937,925,915,929,962,980,977,951,857,0,593,853,919,949,967,977,984],[918,918,921,927,934,941,946,950,951,952,955,960,969,978,983,971,927,809,782,875,926,953,969,979,986],[890,889,895,906,920,933,944,953,961,966,969,968,956,874,971,976,961,933,916,926,946,962,975,983,988],[845,842,853,875,900,922,940,954,965,972,976,973,961,949,972,982,978,968,959,959,965,973,980,986,991],[766,754,782,829,874,909,935,954,967,975,979,979,975,974,982,987,987,983,978,976,977,980,984,988,992],[602,554,640,758,842,896,930,953,967,977,982,984,983,985,988,991,991,989,987,984,983,984,985,987,990],[179,0,311,663,812,886,927,952,968,978,983,986,987,988,990,991,991,990,988,986,983,982,981,982,985],[0,0,0,595,800,884,928,953,969,978,983,986,987,987,987,988,987,986,985,982,978,973,968,966,971],[0,0,45,640,816,892,932,956,970,978,982,983,983,982,981,980,980,978,976,972,966,954,939,928,939],[554,429,570,749,851,907,940,959,970,976,979,978,976,974,971,969,967,965,963,957,947,925,876,794,848],[781,756,782,837,888,924,948,962,970,974,973,971,966,961,956,952,949,946,943,937,923,890,783,0,572],[880,872,877,895,917,937,953,963,968,969,966,960,952,944,935,928,923,920,916,910,897,865,785,605,746],[932,929,929,931,936,944,954,961,964,962,956,946,933,919,906,894,885,881,879,877,870,855,831,821,863],[962,962,960,954,945,941,948,955,957,952,942,927,908,885,862,842,828,822,825,832,840,846,854,872,902],[980,981,979,968,942,916,931,945,948,941,926,904,875,838,797,758,732,728,744,772,803,832,860,888,915],[990,991,989,976,921,777,915,940,941,930,909,877,832,772,694,612,555,556,612,687,756,811,856,891,919],[990,991,990,980,942,916,942,947,940,922,894,850,784,683,528,317,142,183,383,570,700,787,846,888,918],[981,975,975,981,982,978,971,961,944,920,883,828,738,581,274,0,0,0,0,430,646,764,836,883,915],[961,909,923,983,995,994,985,971,950,922,880,816,710,507,16,0,0,0,0,326,612,751,829,879,912],[952,897,946,987,999,999,990,976,955,925,883,819,712,511,33,0,0,0,0,341,617,752,829,878,911],[968,961,976,993,1000,999,992,978,958,930,891,832,741,585,288,0,0,0,71,459,658,769,837,881,912],[983,983,989,996,1000,998,991,979,961,936,901,852,782,676,520,317,170,228,419,593,715,796,850,888,915]],"best":[3.11662,3.15731],"mean":[[823,820,815,808,798,785,769,748,722,690,650,601,548,504,493,525,577,628,670,702,726,743,755,764,770],[831,827,822,813,802,788,769,745,714,675,625,560,480,406,391,453,536,606,658,695,720,738,750,759,766],[839,836,830,820,808,792,770,743,708,662,601,516,400,267,250,378,502,589,648,688,715,733,745,754,760],[849,845,839,829,815,797,774,744,705,653,582,479,321,81,84,325,481,579,641,682,708,726,739,747,754],[859,855,849,838,823,804,779,747,706,651,575,464,289,0,56,322,478,574,636,675,702,719,731,739,745],[869,866,859,848,832,812,786,753,711,656,583,481,338,183,216,372,494,578,633,670,694,710,721,729,735],[879,876,869,857,841,820,793,761,720,669,604,523,429,357,369,446,526,589,635,666,687,700,710,717,723],[887,884,877,865,848,827,800,769,731,686,634,574,516,480,483,520,567,610,643,666,681,691,697,703,709],[893,891,883,870,853,831,806,776,743,705,664,623,588,568,569,586,612,637,657,670,678,683,685,688,693],[895,893,885,871,854,832,808,781,752,722,693,666,645,634,634,643,656,668,676,681,681,679,676,675,678],[892,889,881,867,849,829,806,782,759,737,717,701,691,686,687,692,697,701,701,697,690,680,671,665,665],[883,879,870,856,838,819,798,779,762,747,737,730,728,729,731,734,736,735,730,720,706,690,674,662,658],[867,861,851,837,819,801,784,770,760,755,753,755,759,765,770,773,773,770,761,748,730,709,687,670,662],[843,835,823,808,791,776,764,757,755,759,767,777,787,797,804,808,808,804,795,780,760,736,711,690,678],[812,801,786,769,753,740,735,738,748,762,779,796,812,825,835,841,842,838,828,813,793,769,744,721,705],[774,758,738,718,701,694,699,715,738,764,790,815,835,852,865,872,874,870,861,846,827,804,779,756,738],[729,704,676,650,634,635,656,690,729,767,802,832,857,877,892,901,904,901,892,878,859,837,813,790,771],[678,640,599,562,546,564,611,668,724,772,814,849,877,900,916,927,931,929,920,907,889,867,844,821,801],[622,567,505,450,439,491,574,655,723,780,826,864,895,920,938,949,954,953,945,932,914,893,870,847,826],[568,492,401,324,338,443,558,654,730,790,839,878,911,936,956,968,974,973,965,952,934,913,891,868,847],[527,436,326,253,316,449,570,666,741,801,850,890,923,949,969,983,989,988,980,967,949,928,906,884,863],[513,426,338,311,384,497,600,686,755,812,859,898,931,957,977,991,998,997,989,976,959,938,917,895,874],[529,464,416,417,474,556,637,709,770,822,866,903,934,960,980,993,1000,999,992,979,963,943,923,902,881],[563,523,500,509,551,609,672,731,783,830,870,904,934,958,977,990,996,996,989,977,962,943,924,904,885],[603,579,570,580,611,654,702,750,794,835,871,902,929,952,969,981,987,987,981,971,956,940,921,903,885]],"newest":[-2.25384,14.2432],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432]]}}, +{"best":0.747335,"evaluations":11,"generation":5,"state":{"acquisition":[[969,968,968,967,965,961,954,942,922,894,870,893,938,960,967,970,970,967,965,966,970,976,982,987,991],[961,960,960,959,958,954,947,933,906,847,668,530,871,925,919,932,936,930,928,937,952,965,976,984,990],[949,949,949,950,950,949,944,933,912,867,753,663,907,951,895,887,861,812,834,889,929,954,971,982,989],[934,933,935,938,941,943,943,940,932,920,911,929,968,989,986,956,842,0,516,835,913,949,969,982,990],[912,911,914,921,929,936,942,947,949,952,956,964,976,988,992,977,927,790,751,861,921,953,972,985,993],[879,877,884,896,912,927,941,952,961,968,973,973,962,885,980,983,965,933,913,923,945,964,979,989,996],[825,821,835,860,889,915,937,953,966,975,980,979,967,958,980,989,984,973,963,961,968,977,986,993,998],[730,717,750,806,859,901,931,953,969,979,985,985,982,983,990,996,995,990,984,982,982,986,991,996,1000],[533,477,581,723,823,886,926,953,970,982,988,991,991,993,997,999,999,997,994,991,990,990,992,995,999],[18,0,187,613,790,876,924,953,972,983,990,993,995,996,998,999,1000,998,996,993,990,988,988,989,993],[0,0,0,535,778,876,926,955,973,984,990,993,995,995,995,996,995,994,992,989,984,979,974,973,978],[0,0,0,590,798,886,932,959,975,984,989,991,991,990,989,988,987,985,983,978,971,959,942,932,944],[483,325,497,716,838,904,941,963,976,983,986,985,983,981,978,975,973,971,967,961,949,925,871,781,842],[752,718,749,818,879,922,950,967,976,980,980,977,972,967,961,957,953,950,945,937,921,883,760,0,517],[866,855,861,883,911,937,956,968,974,975,971,965,957,947,938,930,924,919,914,906,890,853,758,547,718],[926,921,920,924,932,945,957,966,969,966,959,948,935,920,904,891,881,875,871,866,857,838,810,800,851],[960,959,956,949,941,941,951,958,960,954,943,926,905,881,855,832,815,808,808,814,820,827,836,858,894],[980,981,978,966,935,911,932,946,947,938,922,897,865,825,780,737,706,698,713,741,775,809,842,876,909],[990,992,991,978,911,755,907,932,932,920,896,861,812,745,660,568,501,497,555,639,718,784,837,879,913],[990,992,994,988,944,861,909,923,918,900,869,820,747,633,460,222,21,55,278,496,651,755,825,875,911],[977,972,982,987,967,928,916,913,903,880,842,780,677,495,138,0,0,0,0,325,586,728,813,869,908],[954,874,918,959,940,902,892,893,884,861,819,748,623,379,0,0,0,0,0,198,546,712,806,865,904],[955,896,863,874,763,730,825,860,863,846,806,735,610,364,0,0,0,0,0,220,553,714,806,864,903],[974,955,928,855,133,185,750,833,850,840,807,746,642,456,100,0,0,0,0,367,603,735,816,868,904],[987,976,955,910,802,718,790,839,854,847,821,774,698,581,404,180,26,102,325,529,672,767,832,877,909]],"best":[3.11662,3.15731],"mean":[[831,827,821,814,803,789,772,751,724,690,649,599,543,497,486,518,572,625,668,701,726,743,756,764,771],[838,834,828,819,808,792,773,748,716,676,624,557,476,399,383,447,533,604,657,694,720,738,751,760,766],[847,843,836,827,813,796,774,746,710,663,600,515,397,262,245,375,500,588,647,687,714,733,746,754,761],[856,852,845,835,820,802,778,747,707,654,582,478,319,81,85,324,480,577,640,681,708,726,738,747,753],[866,862,854,843,828,808,782,749,707,651,574,462,286,0,54,319,476,572,634,674,700,718,729,738,744],[875,872,864,852,836,815,788,755,711,655,580,477,332,174,206,365,489,573,630,667,691,708,718,726,732],[884,881,873,861,844,822,795,761,720,667,600,517,420,346,357,436,518,583,629,661,682,696,706,713,719],[892,888,880,868,850,828,801,769,730,683,629,567,507,468,471,509,558,602,635,659,675,685,692,697,703],[896,893,885,872,854,832,806,775,740,701,659,616,579,557,558,576,602,628,648,663,671,676,679,682,686],[897,894,885,872,854,832,807,779,749,718,687,658,636,625,625,634,647,659,668,673,673,671,668,667,670],[893,889,880,866,848,827,804,779,755,732,711,694,683,678,679,684,689,693,694,690,683,673,664,657,657],[882,878,868,853,835,815,795,775,757,742,731,724,721,722,725,728,730,729,724,714,701,684,668,656,652],[865,858,847,832,815,797,780,765,755,749,748,750,754,760,765,768,769,765,757,744,726,705,684,666,657],[839,831,818,802,785,769,757,750,750,754,762,772,783,793,800,805,805,801,792,777,758,735,710,689,676],[807,794,778,761,744,732,727,731,742,757,775,793,809,823,833,839,840,836,827,812,792,769,744,722,706],[766,748,727,706,690,684,690,708,733,760,788,812,833,850,863,870,872,869,860,846,827,805,781,758,740],[717,690,661,634,619,623,648,685,726,765,800,831,856,876,890,899,903,900,892,879,861,839,816,794,774],[660,620,577,540,528,553,605,666,723,772,814,849,877,899,915,926,930,928,920,908,890,870,848,825,805],[598,539,473,420,421,486,575,657,726,782,828,866,896,920,937,949,954,952,945,933,916,896,874,852,832],[538,456,361,297,340,455,570,663,736,795,842,881,912,937,955,968,973,972,965,953,936,917,896,874,853],[496,400,298,264,353,480,591,681,752,809,855,894,925,950,969,982,988,987,980,968,952,933,912,890,870],[488,406,343,353,435,536,628,705,769,822,867,904,934,959,978,991,997,996,990,978,962,943,923,902,882],[514,462,437,460,522,596,667,731,786,834,875,910,939,963,981,994,1000,1000,993,982,967,949,929,909,890],[558,529,524,547,592,646,702,754,801,843,880,912,940,962,979,991,997,997,991,981,966,950,931,912,894],[604,590,592,612,646,687,730,773,813,849,882,912,937,957,973,984,990,990,985,975,962,947,930,912,895]],"newest":[4.72756,5.29756],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756]]}}, +{"best":0.747335,"evaluations":12,"generation":6,"state":{"acquisition":[[942,941,942,942,942,939,933,921,898,857,806,832,907,939,947,952,952,947,945,949,960,971,981,989,994],[928,927,928,930,931,930,925,912,885,819,590,249,851,913,891,909,911,897,897,918,942,962,977,987,994],[910,909,911,915,919,921,920,911,890,840,689,437,891,945,864,869,824,728,780,871,924,955,974,987,995],[886,884,888,896,905,913,917,916,908,890,868,881,946,979,975,940,810,0,465,827,912,951,973,987,996],[851,849,857,872,888,904,916,923,927,928,930,939,957,978,983,962,888,620,641,840,916,953,975,989,998],[800,796,811,838,868,895,915,931,942,951,957,960,952,877,965,955,908,830,821,881,929,959,978,991,999],[719,710,741,793,844,885,915,937,953,964,971,971,958,918,919,916,885,859,874,909,941,964,980,992,1000],[577,548,622,730,816,876,916,942,961,972,978,977,966,934,868,784,776,831,883,920,947,966,981,991,999],[289,165,399,647,790,870,917,947,966,977,982,980,967,935,840,199,160,755,870,918,945,963,976,987,995],[0,0,0,556,771,868,921,951,970,980,983,979,965,931,839,352,0,683,848,906,936,953,965,976,986],[0,0,0,507,770,873,926,956,972,980,982,975,959,925,850,669,510,704,831,889,918,933,941,952,967],[0,0,0,552,789,885,933,959,974,979,978,969,951,917,858,766,704,752,823,869,892,899,892,893,925],[289,0,299,658,821,899,940,963,974,977,972,960,940,906,857,798,762,777,816,848,862,850,792,672,808],[641,567,630,759,855,913,946,964,972,972,964,949,925,891,847,800,772,777,801,824,830,802,663,0,481],[796,773,788,834,884,924,950,964,968,964,953,934,907,869,824,780,753,754,773,793,799,774,670,361,650],[877,869,873,885,904,928,948,959,961,954,939,916,882,838,786,736,706,705,726,752,768,764,737,723,809],[924,922,923,920,913,920,939,951,951,941,921,892,850,794,727,660,618,617,651,695,732,758,778,811,866],[951,952,954,945,908,882,921,938,937,924,900,863,809,733,636,531,459,463,530,615,689,747,796,843,890],[965,968,972,964,891,705,898,923,922,905,875,828,757,650,494,297,146,170,333,506,638,732,801,856,900],[968,969,978,976,923,792,889,909,905,885,849,791,697,542,274,0,0,0,8,369,584,715,800,860,904],[959,948,965,973,940,873,882,893,886,864,823,754,637,421,0,0,0,0,0,226,538,701,798,861,904],[945,845,898,943,900,829,850,868,866,844,800,725,591,325,0,0,0,0,0,138,515,695,797,861,904],[950,878,846,860,653,588,783,837,845,828,786,710,575,305,0,0,0,0,0,163,526,701,800,862,904],[968,944,912,829,0,57,717,810,830,819,782,714,593,371,0,0,0,0,0,284,567,718,808,865,905],[982,967,941,880,702,585,737,808,828,820,790,733,637,479,214,0,0,0,113,430,624,743,820,871,907]],"best":[3.11662,3.15731],"mean":[[859,855,849,839,826,810,790,765,734,695,648,591,528,475,464,502,562,617,661,694,717,733,744,752,758],[867,863,856,845,831,813,791,762,727,683,626,553,465,382,369,439,527,599,650,686,710,726,737,745,751],[875,870,863,852,836,817,792,761,721,671,604,514,390,252,243,377,499,584,641,679,703,719,730,737,742],[883,878,870,858,842,820,794,760,717,661,586,479,316,81,107,335,484,576,635,673,696,711,721,727,732],[890,885,877,864,846,824,795,760,715,656,575,460,279,0,93,340,486,576,633,668,690,703,711,716,720],[896,892,883,869,851,827,797,761,714,655,576,468,317,166,232,391,507,585,635,665,684,694,700,703,707],[902,897,887,873,854,829,799,762,716,660,589,500,400,338,375,463,544,603,642,666,680,686,688,690,692],[905,900,890,875,855,829,799,763,720,669,610,546,488,463,486,537,588,629,656,672,679,680,678,676,676],[906,900,890,874,853,827,797,762,724,681,636,594,563,556,572,603,635,660,676,683,682,677,669,663,661],[903,897,886,869,847,821,792,760,727,694,663,638,625,627,641,661,680,693,700,699,691,679,665,653,647],[895,889,876,859,836,810,783,755,729,706,688,678,677,684,696,710,722,728,728,720,706,688,667,649,639],[883,875,861,842,819,794,769,747,729,716,711,712,719,731,743,754,761,763,758,746,727,703,677,653,639],[864,854,839,818,795,771,750,735,726,726,731,742,756,770,783,793,798,796,789,774,753,725,695,668,649],[839,826,808,786,762,741,726,719,722,734,750,769,788,805,818,828,831,829,820,804,781,753,721,691,669],[808,791,769,744,719,701,695,701,718,741,767,793,816,835,850,859,862,859,850,834,811,783,751,721,697],[769,746,718,689,664,652,658,681,714,750,784,814,841,862,878,887,891,888,878,862,840,813,783,754,729],[724,692,655,619,594,593,620,664,713,759,800,834,863,886,902,913,917,914,905,889,868,842,814,785,760],[672,628,578,530,508,529,586,654,716,770,815,853,883,907,924,935,940,937,928,913,893,869,842,814,789],[615,555,485,425,415,477,568,653,724,783,830,869,900,925,943,954,959,957,949,934,915,891,866,839,815],[559,480,389,323,354,461,572,664,738,797,845,883,915,940,958,970,975,974,965,951,933,910,886,861,837],[514,423,328,295,372,490,597,684,754,811,858,896,927,952,970,982,988,986,978,965,947,925,902,877,854],[494,411,349,362,444,543,633,709,773,825,869,905,936,960,978,990,996,995,987,974,956,936,914,891,868],[505,448,423,454,522,599,671,735,790,838,878,913,941,965,983,994,1000,999,992,979,963,943,922,900,879],[538,505,502,534,587,647,705,758,806,848,885,917,944,966,983,995,1000,999,992,981,965,947,927,906,886],[578,562,568,597,639,686,733,777,819,856,890,919,944,965,981,991,996,996,990,979,964,947,929,909,890]],"newest":[10.0,3.80158],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158]]}}, +{"best":0.747335,"evaluations":13,"generation":7,"state":{"acquisition":[[905,904,904,906,907,906,900,887,862,813,741,774,881,919,925,926,924,920,924,938,956,972,984,993,1000],[886,885,886,890,893,894,891,878,849,778,511,26,834,900,864,875,872,858,872,907,939,962,978,989,997],[862,860,863,870,878,884,884,876,854,800,623,226,868,928,817,820,754,638,741,858,918,951,970,983,992],[829,826,833,846,861,873,881,882,873,852,821,830,918,956,946,896,724,0,376,806,900,940,961,974,985],[784,779,792,816,842,864,881,891,895,895,896,908,931,954,955,923,822,464,560,811,897,935,953,961,971],[716,709,734,777,820,856,883,901,914,923,931,936,932,853,935,915,850,753,771,855,909,938,949,944,942],[611,594,645,726,797,850,886,912,929,942,950,952,940,894,885,868,819,795,833,885,923,945,952,938,533],[426,377,500,660,774,846,891,921,941,954,960,960,948,912,830,702,675,767,845,896,929,949,956,949,938],[63,0,242,581,754,845,897,929,949,961,966,964,951,915,805,3,0,686,832,893,927,946,955,955,962],[0,0,0,505,744,849,904,936,956,966,969,964,948,910,805,195,0,607,807,879,915,935,944,946,960],[0,0,0,471,748,857,912,943,960,968,968,961,942,902,811,557,293,609,780,855,892,911,917,917,941],[0,0,0,511,767,869,920,948,962,968,965,954,932,891,813,671,559,646,757,824,859,871,861,835,889],[124,0,124,604,796,883,927,951,963,965,959,945,920,877,806,707,637,665,734,787,814,811,752,518,749],[533,410,513,702,826,895,933,953,961,960,951,933,903,857,789,707,651,659,704,745,761,736,586,0,477],[721,682,708,781,852,903,935,951,956,952,939,917,882,830,760,681,626,625,659,694,705,669,499,0,543],[820,806,815,839,870,905,932,947,949,941,924,897,855,794,714,626,563,557,592,632,650,629,546,473,694],[877,872,877,881,878,894,923,938,938,928,906,872,820,746,646,531,447,439,491,554,597,613,613,654,772],[910,910,916,912,871,850,905,925,924,911,884,842,778,682,543,372,238,234,337,455,545,607,660,728,812],[928,930,938,935,858,665,882,909,908,891,859,808,727,598,390,90,0,0,95,331,495,606,691,766,835],[935,933,946,948,887,720,866,891,889,869,833,772,671,495,166,0,0,0,0,193,453,607,711,788,849],[931,911,933,945,899,805,848,870,868,847,806,736,614,384,0,0,0,0,0,69,427,613,725,802,859],[925,809,863,913,846,745,810,843,845,825,783,707,570,296,0,0,0,0,0,14,426,624,738,813,865],[935,856,818,832,538,457,744,811,823,807,765,689,550,269,0,0,0,0,0,63,454,643,752,822,871],[954,925,889,798,0,0,681,783,806,795,758,686,559,316,0,0,0,0,0,190,504,669,767,832,876],[967,949,919,847,601,463,689,775,800,793,761,699,593,409,79,0,0,0,0,339,565,699,784,841,882]],"best":[3.11662,3.15731],"mean":[[878,874,867,856,842,825,802,775,741,699,646,581,505,441,431,482,549,602,637,653,655,647,631,614,599],[885,881,873,862,847,827,803,773,735,688,627,547,445,348,342,431,525,590,628,644,643,629,607,582,560],[892,888,879,867,851,830,803,770,729,676,607,511,376,223,235,387,507,581,621,635,630,610,579,544,513],[899,894,885,872,854,831,803,768,723,666,588,478,307,66,139,362,498,576,616,627,618,591,550,501,456],[904,899,890,876,857,832,802,765,718,657,575,456,269,0,142,370,502,577,614,622,608,573,521,454,387],[909,903,893,878,858,832,801,762,713,652,569,455,294,146,248,411,520,585,616,620,601,560,497,410,313],[911,906,895,879,858,831,798,758,710,650,574,477,367,309,369,470,550,601,625,625,602,556,485,383,256],[912,906,895,877,855,827,793,754,707,652,587,515,452,433,472,535,590,626,642,637,612,564,491,387,254],[910,904,891,873,849,820,786,748,705,657,606,559,528,528,557,597,634,658,666,658,631,585,516,422,311],[905,898,884,864,839,810,776,740,702,664,630,604,594,603,626,654,679,694,696,685,658,615,554,476,392],[896,887,872,851,825,795,763,730,699,673,654,646,649,664,684,706,722,731,730,717,691,651,598,535,471],[882,872,855,832,805,775,745,717,696,682,678,684,697,715,734,751,763,768,764,750,725,690,644,592,541],[863,850,831,807,778,749,722,702,692,692,702,718,737,758,777,792,801,804,798,784,761,728,688,644,602],[838,822,800,773,743,715,694,685,689,703,724,748,773,795,814,828,836,837,830,816,794,765,729,691,654],[807,787,761,730,698,673,661,666,686,714,745,776,804,828,847,860,867,867,861,847,826,799,767,732,700],[770,744,712,676,642,622,625,649,686,727,766,801,831,856,875,888,895,895,888,874,855,830,800,769,739],[726,693,651,608,573,564,590,637,690,741,785,823,855,881,900,913,920,919,912,899,880,857,829,800,772],[677,632,579,525,491,506,563,633,699,756,804,844,876,902,922,935,941,941,934,921,903,880,855,827,801],[622,563,494,429,410,464,553,639,712,772,821,862,894,920,940,953,959,959,952,940,922,900,876,850,824],[567,492,407,343,363,459,565,656,730,789,837,877,910,936,955,968,974,974,967,955,938,917,893,869,844],[519,433,347,317,384,493,595,680,749,806,852,891,922,948,967,980,986,986,979,967,950,930,907,884,860],[490,409,351,369,450,545,633,707,769,821,865,902,932,956,975,988,994,994,987,975,959,940,918,895,873],[488,429,409,448,521,599,670,733,788,835,876,910,939,962,980,993,999,998,992,981,965,947,926,904,882],[510,477,480,522,582,645,703,757,805,847,884,916,943,965,983,994,1000,1000,994,983,968,951,931,910,889],[545,531,544,582,631,682,731,777,819,857,890,920,945,966,982,993,998,998,993,983,969,952,933,914,894]],"newest":[10.0,0.0],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0]]}}, +{"best":0.747335,"evaluations":14,"generation":8,"state":{"acquisition":[[903,902,904,909,914,916,913,902,875,820,720,754,898,944,946,946,941,931,929,940,954,960,950,889,0],[879,877,881,889,898,904,904,894,865,791,494,0,861,930,879,891,882,854,862,903,938,956,957,923,807],[846,844,852,865,880,892,897,893,872,816,620,20,890,955,812,841,761,565,692,853,924,957,967,955,938],[804,800,814,837,862,882,895,899,892,869,826,824,935,980,966,910,722,0,327,818,919,959,974,971,968],[744,739,762,801,841,874,897,910,915,914,913,923,951,978,977,937,808,249,528,831,923,962,978,976,972],[656,646,689,757,820,868,901,923,937,946,953,960,957,877,957,926,831,677,740,865,932,965,978,972,949],[519,493,581,702,799,864,907,936,954,967,975,978,968,918,905,870,783,741,811,889,938,966,976,964,161],[282,203,408,636,780,864,915,947,968,981,989,988,976,936,843,666,577,713,828,897,940,963,972,958,922],[0,0,116,563,767,868,924,958,979,991,996,993,978,939,817,0,0,634,818,893,933,956,963,952,954],[0,0,0,500,763,876,934,967,987,997,999,994,976,934,814,46,0,560,794,877,919,940,945,935,952],[0,0,0,475,771,886,943,975,992,1000,999,991,969,924,815,466,69,553,765,852,895,914,913,892,928],[0,0,0,511,790,898,952,981,995,1000,997,984,959,910,811,604,424,578,735,818,859,872,851,780,865],[0,0,0,594,815,911,959,984,996,998,991,974,944,892,798,648,531,591,702,776,812,811,741,358,715],[448,250,421,689,842,922,964,986,994,993,982,961,926,868,775,646,551,577,659,725,755,735,579,0,497],[675,612,661,771,865,928,966,984,990,985,970,944,902,837,738,611,519,529,600,664,690,658,464,0,542],[797,774,794,837,882,928,963,980,982,974,955,923,873,796,683,542,437,440,516,589,624,603,484,333,682],[867,860,873,886,888,914,953,971,971,960,936,898,837,744,603,423,283,287,391,497,560,576,557,594,768],[910,908,921,924,882,864,936,958,957,942,914,868,794,676,489,227,5,21,207,384,501,570,620,701,816],[934,934,949,952,873,675,913,941,940,922,889,834,744,592,325,0,0,0,0,253,451,574,666,756,846],[946,940,959,967,900,697,892,921,919,900,862,798,689,493,100,0,0,0,0,117,415,585,699,789,865],[949,917,947,965,904,777,866,897,897,876,835,763,635,389,0,0,0,0,0,8,399,600,723,811,878],[952,820,879,933,836,698,823,867,872,853,810,732,591,306,0,0,0,0,0,0,408,620,744,827,887],[966,882,843,858,470,374,757,833,848,832,790,711,566,272,0,0,0,0,0,22,441,644,763,840,895],[985,953,913,819,0,0,696,804,828,817,778,703,566,300,0,0,0,0,0,143,492,673,781,852,902],[999,979,945,860,523,372,689,790,817,810,775,708,588,374,0,0,0,0,0,288,552,704,799,863,908]],"best":[3.11662,3.15731],"mean":[[895,890,882,871,855,835,811,781,745,700,644,574,491,418,406,462,532,586,616,626,618,595,560,519,482],[901,896,887,875,858,837,810,778,738,688,625,541,434,329,322,418,513,577,612,621,610,582,540,487,436],[907,901,892,878,860,837,809,774,731,676,605,507,369,210,225,381,501,573,609,618,604,571,520,453,384],[911,906,896,881,862,837,807,770,723,665,586,475,304,67,144,363,497,573,609,617,600,562,502,421,328],[915,909,898,883,862,836,803,764,716,654,570,451,263,0,149,374,504,578,612,618,599,557,490,394,273],[917,911,900,883,861,833,799,758,708,644,560,443,277,125,243,412,523,589,619,623,602,557,486,380,236],[918,911,899,881,857,828,793,750,700,637,558,457,340,281,356,468,553,607,632,633,611,565,493,386,238],[917,909,896,877,852,821,784,742,692,634,565,488,420,405,456,529,591,632,651,649,626,581,512,413,282],[913,905,890,870,843,810,773,731,685,633,579,529,497,502,541,590,634,663,676,671,647,604,541,454,349],[906,897,881,859,830,797,759,719,677,636,599,573,565,580,611,647,678,698,705,697,674,634,577,503,420],[895,885,868,843,813,778,742,705,671,642,623,616,624,644,671,699,721,735,737,727,704,667,616,553,487],[881,868,849,822,790,755,720,689,665,651,648,657,675,698,723,745,762,770,770,758,735,701,656,602,548],[861,847,824,795,761,726,694,671,660,661,674,695,719,744,767,786,799,805,802,789,767,735,694,648,601],[837,819,793,761,724,689,663,652,656,674,700,729,757,783,806,823,834,837,832,819,798,768,731,689,649],[807,785,754,717,678,646,629,634,657,689,725,759,791,818,840,855,865,867,861,848,827,799,765,727,691],[772,743,707,664,622,594,593,619,661,706,749,787,820,848,869,884,892,893,887,874,853,827,795,761,727],[730,694,649,598,554,538,562,612,670,724,771,812,846,873,894,909,917,917,910,897,877,852,823,791,760],[683,637,580,520,477,484,541,614,683,743,793,834,868,896,916,931,938,938,931,918,899,874,847,817,787],[631,572,502,433,405,452,540,627,701,762,813,854,888,915,935,949,956,956,949,935,917,894,867,839,811],[577,504,421,357,368,456,559,649,722,782,831,871,904,931,951,964,971,970,963,950,932,910,884,858,831],[527,443,359,331,392,494,593,676,744,800,847,886,918,943,963,976,983,982,975,962,945,923,899,873,847],[490,408,350,371,452,546,631,704,766,818,861,898,929,953,972,985,991,991,984,971,954,934,910,886,861],[477,413,394,440,518,598,669,732,786,833,873,908,937,961,979,991,997,996,990,978,961,941,919,896,872],[488,451,458,509,576,643,702,756,804,846,883,916,943,965,982,994,1000,999,993,981,966,947,926,903,881],[517,500,520,567,624,680,731,777,819,857,891,921,947,968,984,995,1000,999,993,983,968,950,930,908,887]],"newest":[0.890363,6.32204],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0],[0.890363,6.32204]]}}, +{"best":0.747335,"evaluations":15,"generation":9,"state":{"acquisition":[[853,850,854,861,868,873,870,856,823,751,611,655,868,930,932,927,918,903,900,914,931,940,930,860,0],[819,816,822,834,847,856,857,844,809,712,309,0,821,917,853,857,839,798,809,865,910,935,937,899,731],[775,771,782,802,823,839,846,840,811,733,441,0,858,944,738,799,677,372,574,800,893,934,948,935,911],[716,711,730,763,797,823,839,841,827,788,720,749,919,972,951,879,627,0,76,753,886,937,956,954,951],[634,626,659,715,768,809,835,847,846,836,837,881,942,972,961,905,721,0,330,767,889,940,961,960,958],[514,497,559,654,737,796,832,852,860,865,882,918,942,869,935,885,737,489,618,807,898,943,961,957,934],[324,282,410,578,705,784,831,855,865,872,893,925,937,894,871,799,649,589,718,836,905,943,959,949,0],[0,0,171,487,675,777,831,856,860,857,872,912,930,900,791,496,293,555,743,846,905,939,952,941,904],[0,0,0,387,653,775,833,853,842,799,795,882,916,888,734,0,0,456,734,841,897,928,940,930,938],[0,0,0,305,646,781,838,849,807,609,536,844,896,866,704,0,0,372,705,821,879,908,918,904,934],[0,0,0,283,660,796,848,849,763,0,199,821,873,839,690,129,0,368,669,790,848,875,876,846,902],[0,0,0,349,694,817,863,856,752,130,585,820,851,810,678,366,126,400,630,747,804,824,800,693,823],[0,0,0,472,738,842,880,871,797,682,759,827,831,781,660,445,296,416,586,692,745,748,665,128,650],[248,0,233,602,783,865,896,890,853,818,825,832,810,748,629,453,334,397,529,626,671,651,461,0,430],[553,451,544,709,821,882,907,907,889,868,852,829,787,710,582,412,295,334,450,545,586,548,290,0,468],[715,676,716,790,848,887,912,918,908,888,860,820,759,662,512,320,185,214,338,446,496,466,286,37,611],[810,793,818,850,860,872,907,922,914,892,856,803,724,600,410,161,0,8,174,324,408,421,377,431,708],[867,860,880,894,853,800,894,918,911,886,844,780,682,523,265,0,0,0,0,176,329,411,471,590,766],[902,895,917,927,833,517,877,908,900,873,826,752,632,427,61,0,0,0,0,7,265,421,542,672,803],[922,907,933,946,849,596,858,889,883,854,803,719,577,317,0,0,0,0,0,0,224,443,595,721,828],[932,884,921,944,851,684,826,862,859,831,777,684,522,206,0,0,0,0,0,0,214,473,635,754,845],[943,779,844,909,755,570,769,825,830,805,750,653,476,117,0,0,0,0,0,0,236,507,668,778,858],[963,865,815,826,246,141,686,782,799,780,727,629,448,80,0,0,0,0,0,0,287,545,697,797,869],[984,947,899,780,0,0,604,742,773,759,711,617,446,110,0,0,0,0,0,0,356,585,722,814,878],[1000,976,934,826,337,120,586,720,756,747,704,620,469,193,0,0,0,0,0,92,434,626,747,828,886]],"best":[3.11662,3.15731],"mean":[[920,913,903,890,872,850,823,790,751,703,643,569,482,401,384,442,517,574,608,620,615,595,562,520,476],[923,916,906,891,872,848,819,784,742,689,622,536,425,314,303,402,500,567,604,617,609,584,543,490,431],[926,919,908,892,871,846,815,777,731,674,599,499,360,201,215,371,491,565,603,614,604,574,525,459,379],[928,920,908,891,870,842,809,769,719,657,576,462,292,68,150,360,491,567,605,615,601,567,510,429,323],[929,921,908,890,866,837,802,759,706,640,553,429,239,0,163,375,501,574,610,618,602,563,500,406,267],[928,920,906,886,861,830,793,748,693,624,534,410,235,103,248,414,523,587,619,624,606,565,497,393,227],[926,917,902,881,854,821,782,735,678,608,521,410,288,250,350,467,553,607,633,636,616,574,505,398,224],[922,912,896,873,845,810,769,720,663,596,517,432,367,373,444,526,590,633,653,652,631,589,522,421,265],[915,904,887,863,832,796,753,703,648,587,524,470,447,471,525,583,631,663,677,674,652,612,549,459,332],[906,894,875,849,816,778,733,685,634,583,540,516,520,550,594,638,673,697,706,700,678,639,582,504,404],[893,880,859,831,796,755,711,665,621,585,564,564,583,616,653,688,715,731,736,728,706,670,619,552,473],[877,861,838,808,770,728,684,643,610,592,593,610,639,672,705,733,754,766,767,758,736,703,657,600,535],[857,838,812,779,739,696,654,621,604,605,624,653,687,720,750,774,791,799,798,787,767,735,694,644,590],[832,810,781,743,700,656,620,600,602,623,656,693,729,762,789,810,824,830,828,816,796,767,729,686,639],[802,776,742,700,653,611,584,583,606,644,687,729,766,798,824,843,854,859,855,843,824,797,762,723,682],[768,736,696,647,597,559,549,573,617,668,717,761,799,830,854,871,882,885,880,869,850,824,792,757,720],[727,689,640,584,532,505,522,573,634,693,745,790,827,857,880,897,906,908,903,891,873,849,820,787,753],[682,634,575,511,461,459,511,585,656,719,772,816,852,881,903,919,927,929,924,912,894,871,844,813,782],[631,572,502,433,399,437,520,607,682,744,796,839,874,902,923,938,946,947,941,930,913,891,865,836,807],[578,506,426,365,372,451,548,636,708,768,818,859,893,919,940,954,961,962,956,945,928,907,883,856,829],[526,444,366,340,396,492,586,668,735,791,838,877,909,934,954,967,974,975,969,958,942,922,898,873,847],[483,402,349,371,452,544,627,699,760,812,855,892,922,946,965,978,984,985,979,968,953,933,911,887,862],[460,396,381,432,513,594,666,729,783,830,870,904,933,956,973,986,992,992,987,976,961,943,922,899,875],[462,424,437,496,569,639,701,755,804,846,883,915,941,963,980,991,997,997,992,982,968,950,930,908,886],[484,470,497,553,616,676,730,778,821,860,893,923,948,968,984,994,1000,1000,995,985,972,955,936,916,894]],"newest":[-5.0,15.0],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0],[0.890363,6.32204],[-5.0,15.0]]}}, +{"best":0.0426524,"evaluations":16,"generation":10,"state":{"acquisition":[[855,852,855,862,869,873,870,855,818,737,583,643,880,950,954,948,937,920,916,929,946,956,945,869,0],[817,813,819,831,845,854,854,840,800,689,219,0,826,936,870,870,849,804,814,873,922,949,952,912,735],[767,763,774,795,817,835,842,835,803,714,378,0,872,968,744,798,657,321,545,796,901,947,964,952,931],[702,695,715,750,787,817,834,838,823,781,710,753,942,1000,976,891,594,0,0,739,891,950,973,973,974],[610,599,635,695,754,800,830,845,846,839,845,898,968,1000,985,920,707,0,242,754,896,954,978,980,982],[474,454,521,626,719,785,828,852,863,873,897,939,968,897,954,895,726,444,586,802,906,957,979,976,958],[260,211,351,539,682,773,827,857,871,883,909,946,961,914,881,796,625,557,702,836,914,957,976,968,0],[0,0,77,434,647,764,827,857,866,866,886,932,952,918,790,441,214,516,731,849,915,953,970,959,927],[0,0,0,319,622,762,829,854,845,802,800,897,935,903,724,0,0,398,720,843,907,942,957,948,963],[0,0,0,226,616,769,835,849,804,588,500,854,912,877,691,0,0,302,689,821,887,921,933,921,957],[0,0,0,207,634,786,845,846,750,0,109,827,885,846,677,41,0,303,649,786,853,885,887,857,922],[0,0,0,292,674,811,860,850,727,0,560,826,859,814,665,317,47,348,607,739,804,827,803,688,836],[0,0,0,440,728,839,877,863,774,648,754,833,836,781,647,410,243,372,560,679,738,743,652,59,643],[197,0,182,590,781,864,893,882,838,808,825,836,813,745,614,422,290,355,499,605,655,632,419,0,398],[537,417,525,711,826,882,902,898,881,865,853,832,787,702,563,378,250,288,413,515,559,514,220,0,441],[717,668,714,801,859,887,902,909,903,886,860,820,754,649,486,278,132,158,288,404,457,419,212,0,598],[822,799,825,866,876,865,890,912,910,890,855,799,714,580,374,105,0,0,105,266,356,367,319,388,704],[885,873,892,913,872,763,869,908,906,883,840,772,665,492,212,0,0,0,0,97,264,355,427,567,767],[922,913,930,945,841,309,852,898,894,867,817,737,606,382,0,0,0,0,0,0,190,368,509,659,807],[941,926,943,963,842,549,839,879,874,845,790,697,540,253,0,0,0,0,0,0,144,395,569,714,834],[942,904,914,958,847,676,810,850,847,818,758,656,474,119,0,0,0,0,0,0,134,430,615,750,852],[916,766,780,916,724,549,748,809,815,788,727,618,418,11,0,0,0,0,0,0,162,470,653,777,867],[826,124,818,834,59,49,650,759,780,758,699,589,384,0,0,0,0,0,0,0,223,515,685,798,878],[366,515,848,764,0,0,548,712,748,734,680,575,381,0,0,0,0,0,0,0,304,561,714,816,888],[0,695,851,760,118,0,521,685,729,721,673,578,408,96,0,0,0,0,0,15,394,608,742,832,897]],"best":[3.23585,2.19754],"mean":[[928,921,910,896,878,855,828,794,754,705,645,570,481,400,382,440,515,572,606,619,615,596,563,521,475],[931,923,912,897,877,853,824,788,744,691,623,535,423,312,302,400,498,565,603,616,609,585,545,492,430],[933,925,913,897,876,850,819,780,733,675,599,497,357,199,215,370,490,563,602,614,604,576,528,462,379],[934,925,913,896,874,846,812,771,721,658,575,459,287,66,154,361,490,566,604,614,602,569,513,433,324],[934,925,912,893,870,840,805,761,708,640,551,425,233,0,169,377,501,573,609,617,603,565,504,410,269],[932,923,909,889,864,833,795,749,693,623,531,404,228,103,253,416,523,587,618,624,607,567,501,398,229],[929,919,904,883,856,823,784,736,678,607,517,403,281,249,352,469,553,606,632,635,616,575,508,402,224],[924,913,897,875,846,812,770,721,663,594,513,426,361,371,444,526,590,632,652,652,631,590,524,424,263],[917,905,887,863,833,797,754,705,648,585,520,464,442,468,524,583,630,662,676,673,652,612,550,460,328],[906,893,874,848,817,779,735,686,634,582,537,511,516,547,592,636,672,695,704,698,677,639,582,504,400],[893,878,857,830,796,756,713,667,622,585,562,560,580,613,651,685,712,729,734,726,705,669,618,551,469],[876,859,836,806,770,730,687,647,613,593,592,608,636,669,702,730,751,763,765,755,734,701,656,598,532],[854,835,809,776,738,697,658,626,608,608,624,652,684,717,747,771,787,796,795,785,764,734,693,643,588],[828,805,776,739,699,659,626,607,608,626,657,692,727,759,786,807,821,827,824,813,794,765,728,684,637],[798,770,735,694,652,615,592,593,614,649,688,728,764,795,821,839,851,855,852,841,822,795,761,722,681],[762,727,686,639,595,565,561,584,626,673,719,761,797,827,851,868,878,881,877,866,848,822,792,757,720],[720,677,626,572,528,512,536,585,643,698,747,790,826,855,878,893,903,905,900,889,871,848,819,787,754],[674,619,555,492,453,467,525,597,664,724,774,816,851,879,901,916,924,926,921,910,893,870,844,814,783],[624,553,472,400,385,444,532,617,689,749,798,839,873,900,921,935,943,944,939,928,911,890,865,838,809],[574,486,385,317,352,453,557,644,715,772,820,860,892,918,938,951,958,959,954,943,928,907,884,858,831],[532,429,321,289,376,491,592,674,740,795,840,877,908,933,952,965,972,972,967,957,941,922,900,876,850],[508,406,322,339,437,542,631,704,765,815,857,893,922,946,963,976,982,983,978,967,953,935,913,890,866],[508,426,382,421,506,593,669,733,787,833,873,906,933,956,973,984,990,991,986,976,962,945,925,903,880],[529,474,459,500,569,639,703,759,807,849,886,916,942,963,979,990,996,996,992,982,969,953,933,913,891],[562,530,530,566,621,679,733,782,825,863,897,925,949,969,984,994,1000,1000,995,987,974,958,940,921,900]],"newest":[3.23585,2.19754],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0],[0.890363,6.32204],[-5.0,15.0],[3.23585,2.19754]]}}, +{"best":0.0426524,"evaluations":17,"generation":11,"state":{"acquisition":[[816,809,813,824,836,842,839,817,763,633,341,445,863,961,929,882,879,873,887,921,953,969,959,864,0],[762,754,763,781,802,816,819,800,743,578,0,0,792,948,759,663,695,658,733,847,923,962,968,917,602],[693,682,699,731,765,792,804,796,752,623,115,0,788,951,0,686,389,0,330,749,897,960,982,965,923],[599,586,615,670,726,770,797,802,780,719,604,633,904,960,958,871,437,0,0,671,883,961,992,991,988],[469,448,503,596,686,752,795,814,813,799,801,879,975,964,993,909,584,0,0,677,884,964,998,1000,1000],[278,237,345,507,644,739,797,827,839,846,875,939,985,0,959,871,583,0,323,731,893,966,998,997,959],[0,0,110,400,606,730,801,837,852,860,890,942,953,676,837,717,385,197,522,774,901,965,994,986,0],[0,0,0,278,574,728,808,843,848,838,851,903,908,784,599,133,0,144,582,792,900,958,984,973,911],[0,0,0,157,558,734,816,843,823,748,698,827,880,801,445,0,0,0,580,787,888,943,966,953,973],[0,0,0,75,563,750,827,839,770,440,110,763,861,794,460,0,0,0,549,762,862,915,933,912,967],[0,0,0,79,594,774,841,833,688,0,0,760,844,777,504,0,0,0,508,719,820,868,873,818,919],[0,0,0,185,646,805,857,832,627,0,414,787,827,754,527,0,0,54,460,660,758,795,766,575,806],[0,0,0,354,708,837,874,839,676,484,699,809,808,725,523,154,0,105,403,585,674,689,576,0,579],[0,0,0,527,770,865,888,855,777,752,799,818,785,687,492,191,0,91,325,490,569,548,280,0,343],[421,209,387,671,822,883,895,875,850,842,838,816,756,638,433,144,0,7,212,373,443,390,0,0,391],[658,567,641,781,859,883,888,892,890,876,850,803,719,575,339,18,0,0,48,226,306,252,0,0,554],[796,751,791,862,877,842,867,902,906,885,845,779,671,493,201,0,0,0,0,43,167,169,62,160,675],[879,853,881,921,867,665,840,903,905,879,828,746,613,387,1,0,0,0,0,0,42,148,227,439,752],[928,908,932,961,814,0,826,894,892,861,802,705,542,256,0,0,0,0,0,0,0,173,359,581,803],[954,927,947,982,790,337,814,871,868,834,768,656,463,102,0,0,0,0,0,0,0,224,459,665,837],[955,897,898,974,787,535,775,835,835,800,729,604,381,0,0,0,0,0,0,0,0,287,534,720,862],[923,696,690,920,591,356,694,782,792,760,687,554,308,0,0,0,0,0,0,0,0,355,594,760,882],[803,0,794,819,0,0,564,716,745,720,649,512,257,0,0,0,0,0,0,0,51,423,643,791,897],[161,337,847,738,0,0,425,650,701,686,619,487,241,0,0,0,0,0,0,0,165,490,686,817,910],[0,656,857,730,0,0,372,607,670,663,604,483,263,0,0,0,0,0,0,0,282,554,724,839,922]],"best":[3.23585,2.19754],"mean":[[954,945,933,916,896,871,842,806,762,710,645,566,473,388,371,431,508,567,603,619,617,600,569,525,471],[954,945,932,915,893,867,836,797,751,694,622,531,415,302,293,393,492,560,599,615,611,590,552,497,426],[953,943,930,912,889,862,828,788,738,677,598,493,349,191,212,366,484,558,599,613,607,581,536,470,376],[951,941,926,908,884,855,820,777,725,659,573,454,280,64,157,358,486,561,601,614,605,575,523,444,320],[948,937,922,902,877,847,810,765,710,641,549,421,226,0,173,375,498,570,607,618,606,573,515,423,266],[944,932,916,895,868,836,798,751,694,623,529,400,220,97,251,414,520,585,618,626,612,576,514,413,225],[937,925,908,886,858,824,784,736,678,606,515,399,272,239,348,466,551,605,633,638,622,585,521,417,219],[929,916,898,874,845,810,768,719,661,593,511,420,351,361,438,522,588,631,653,655,637,600,537,437,256],[919,905,885,860,829,792,750,701,645,583,517,459,433,458,516,578,627,661,677,676,657,620,561,470,320],[907,891,869,842,809,771,728,681,630,579,533,505,506,537,584,630,668,693,704,700,681,645,590,510,391],[891,873,850,821,786,747,704,660,617,581,557,553,570,603,642,678,707,726,733,727,707,674,624,554,460],[872,852,826,794,758,718,676,638,607,589,586,600,626,659,692,722,745,759,762,754,735,704,658,599,522],[849,826,797,763,724,683,646,617,601,602,617,643,674,707,737,762,780,790,791,782,763,734,693,641,578],[823,796,763,724,683,644,613,598,601,620,650,683,717,748,776,797,812,820,819,810,791,764,727,681,628],[792,760,721,678,635,599,580,584,608,642,681,719,754,785,810,829,842,847,845,836,818,792,758,718,672],[756,717,672,622,577,549,550,577,620,667,712,752,787,816,840,858,869,873,870,860,843,819,788,752,712],[716,668,612,555,511,499,528,580,638,693,740,781,816,844,867,883,893,896,892,882,866,843,815,782,746],[671,611,542,476,439,458,521,593,661,719,767,808,842,869,890,905,914,916,913,903,887,865,839,809,777],[624,548,462,386,374,440,531,615,686,744,792,832,865,891,911,925,933,935,931,921,906,886,861,833,803],[577,484,377,306,346,453,557,643,712,769,815,853,885,910,929,942,949,951,947,937,922,903,881,855,827],[537,430,316,281,372,491,592,673,739,792,836,872,902,926,944,957,963,965,960,951,937,919,898,873,847],[513,406,315,330,433,541,631,704,764,814,855,889,917,940,957,969,975,976,972,963,950,933,912,890,865],[510,423,373,411,501,591,669,734,788,834,872,904,931,952,968,979,985,986,982,973,961,944,925,904,881],[528,467,448,490,563,638,704,761,809,851,887,917,942,962,977,988,993,994,990,982,970,954,936,916,894],[558,521,519,557,617,678,735,785,829,867,900,928,951,970,984,994,999,1000,996,988,977,962,945,926,906]],"newest":[9.09776,2.7362],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0],[0.890363,6.32204],[-5.0,15.0],[3.23585,2.19754],[9.09776,2.7362]]}}, +{"best":0.0426524,"evaluations":18,"generation":12,"state":{"acquisition":[[816,813,819,832,846,855,854,838,793,686,441,527,881,961,930,879,871,875,904,944,977,992,980,881,0],[769,765,776,797,819,835,839,826,779,647,37,0,826,950,793,679,698,692,784,887,950,980,980,911,659],[708,702,722,755,789,816,828,823,787,682,271,0,819,948,0,662,370,0,455,789,911,962,977,957,935],[626,618,651,705,758,799,823,827,808,754,651,671,903,946,927,833,381,0,0,667,866,934,957,984,991],[512,498,555,644,725,784,821,836,833,817,813,877,959,936,960,873,555,0,0,657,850,914,896,981,1000],[344,315,421,570,691,773,821,846,853,854,876,931,969,0,932,844,583,177,443,746,878,935,950,962,964],[80,9,222,481,659,765,824,853,862,866,890,936,946,719,833,714,439,371,624,810,908,960,988,986,0],[0,0,0,380,633,763,829,857,858,846,859,908,913,809,652,231,0,341,676,836,920,969,994,988,839],[0,0,0,279,617,766,835,856,837,770,738,852,891,825,542,0,0,223,674,835,916,961,983,973,903],[0,0,0,206,619,778,843,853,794,515,295,801,875,820,552,0,0,144,647,815,896,938,954,936,891],[0,0,0,201,641,797,854,849,731,0,0,793,860,804,582,0,0,170,610,779,860,898,901,852,828],[0,0,0,282,681,821,868,850,690,0,497,809,843,782,594,143,0,229,566,729,807,835,803,638,677],[0,0,0,418,731,846,882,857,731,561,728,823,824,754,584,273,56,257,512,664,737,743,630,0,360],[80,0,39,563,781,869,894,872,808,776,812,829,802,719,552,294,118,233,441,583,648,625,371,0,66],[464,280,435,688,824,884,900,888,865,851,845,826,775,673,495,244,71,150,340,484,545,499,149,0,235],[673,595,660,784,855,883,896,902,896,880,855,814,741,616,409,126,0,0,196,360,435,393,130,0,483],[795,756,793,856,870,849,880,909,908,888,852,793,699,542,284,0,0,0,0,210,324,332,241,267,634],[869,847,874,909,861,707,859,908,908,883,838,766,649,450,108,0,0,0,0,29,225,314,363,495,722],[913,896,919,946,822,133,845,899,897,869,817,732,590,340,0,0,0,0,0,0,146,326,459,612,777],[936,912,932,966,813,429,831,880,878,848,790,693,526,213,0,0,0,0,0,0,99,356,531,682,814],[937,885,891,960,807,584,798,850,851,821,760,653,462,86,0,0,0,0,0,0,90,395,586,728,839],[907,713,730,916,646,436,732,808,817,790,728,615,407,0,0,0,0,0,0,0,119,440,630,762,859],[801,0,811,835,0,0,632,757,781,760,699,584,370,0,0,0,0,0,0,0,180,487,667,787,874],[244,445,856,768,0,0,526,707,748,734,677,566,359,0,0,0,0,0,0,0,260,536,700,809,886],[0,695,865,760,0,0,484,673,724,716,665,562,376,27,0,0,0,0,0,0,348,584,729,827,897]],"best":[3.23585,2.19754],"mean":[[942,935,924,909,890,866,837,803,761,710,645,564,465,372,360,436,517,570,592,586,558,514,471,453,472],[943,935,924,908,888,863,832,795,751,695,624,531,409,284,285,404,506,565,587,577,539,478,413,381,411],[944,935,923,906,885,858,826,787,739,679,601,495,345,174,214,384,501,564,585,570,522,443,346,291,338],[943,934,921,903,880,852,818,777,726,662,577,458,279,52,175,382,504,567,585,566,510,414,277,180,258],[941,932,918,899,875,845,809,765,712,644,554,425,225,0,198,399,515,573,589,567,507,399,230,72,190],[938,928,913,893,867,836,798,752,696,626,533,403,217,97,270,433,533,585,598,574,514,407,236,62,171],[933,922,906,885,857,824,785,737,680,609,519,400,267,238,358,478,559,602,612,589,532,435,293,162,216],[927,915,897,874,845,810,769,721,664,595,513,420,348,360,443,529,591,626,632,610,560,478,370,278,291],[918,905,886,861,830,794,751,703,648,585,518,458,431,458,519,582,628,654,657,637,594,527,444,376,369],[906,892,871,845,812,774,731,683,633,581,534,505,506,538,586,632,667,686,687,668,631,576,511,457,440],[892,876,853,824,789,750,707,662,619,583,558,554,571,605,644,680,707,720,719,702,669,623,571,524,503],[874,855,830,799,762,721,679,641,609,591,588,602,628,662,695,724,745,754,751,735,707,667,623,582,558],[852,831,802,768,728,687,649,620,604,605,620,646,678,710,740,765,781,787,783,768,742,708,669,632,607],[827,801,769,730,688,648,616,601,604,623,653,687,721,753,780,801,814,818,813,799,776,745,710,677,651],[796,766,728,685,640,603,583,587,610,646,685,723,759,789,815,833,844,847,842,828,807,779,747,716,690],[761,724,679,629,583,553,552,580,623,670,716,756,792,822,845,862,872,874,868,855,835,809,780,750,726],[722,675,620,562,516,502,530,583,641,696,744,786,821,850,872,888,896,898,892,879,860,836,809,781,757],[678,619,550,483,443,460,522,596,664,722,771,813,847,874,896,910,918,919,913,901,883,860,834,808,785],[631,556,470,393,377,441,533,618,689,748,796,836,869,896,916,930,937,938,932,920,902,881,857,832,809],[584,492,385,312,348,454,559,645,715,772,819,857,889,914,934,947,953,954,948,936,920,900,877,853,831],[544,437,320,284,374,493,594,675,741,794,839,876,906,930,948,961,967,967,962,950,935,915,894,871,849],[519,410,316,331,435,543,633,706,766,815,857,892,920,943,961,973,979,979,973,962,947,929,908,887,866],[515,424,371,412,503,593,670,734,789,834,873,905,933,954,971,982,988,988,982,972,958,940,921,900,880],[531,467,446,490,565,639,704,761,809,851,887,917,943,963,979,990,995,995,990,980,966,950,931,911,891],[559,520,517,557,617,679,735,784,828,866,899,927,951,970,985,995,1000,1000,995,986,973,957,939,920,901]],"newest":[9.79007,2.3748],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0],[0.890363,6.32204],[-5.0,15.0],[3.23585,2.19754],[9.09776,2.7362],[9.79007,2.3748]]}}, +{"best":0.0426524,"evaluations":19,"generation":13,"state":{"acquisition":[[761,757,769,791,813,830,833,818,768,641,318,414,883,978,937,861,848,850,887,939,980,1000,991,880,0],[700,695,714,748,782,808,818,806,756,604,0,0,828,970,779,611,630,614,737,869,949,989,996,923,0],[621,614,646,698,749,788,809,805,766,645,150,0,822,970,0,621,243,0,317,749,901,968,995,958,563],[516,504,557,639,716,772,805,813,791,724,584,599,913,971,942,825,274,0,0,592,840,930,973,973,751],[368,348,438,570,683,761,806,825,819,793,778,862,975,961,978,869,463,0,0,567,806,883,876,955,829],[151,108,274,489,651,754,812,838,842,836,855,928,984,0,943,828,475,0,280,671,831,890,887,987,787],[0,0,36,398,625,752,820,850,854,848,871,934,955,659,821,657,258,163,516,751,867,923,957,983,0],[0,0,0,301,607,757,830,858,852,825,829,898,912,778,592,21,0,142,588,786,885,938,966,955,451],[0,0,0,215,603,768,841,861,832,734,668,830,884,801,451,0,0,21,593,789,883,931,950,912,712],[0,0,0,164,616,786,854,861,789,432,78,775,867,797,464,0,0,0,568,770,863,907,913,841,728],[0,0,0,176,645,809,868,858,720,0,0,773,851,781,500,0,0,0,528,733,827,865,848,701,655],[0,0,0,261,689,835,882,856,661,0,417,794,833,756,514,0,0,30,478,680,773,799,739,361,477],[0,0,0,394,739,860,894,857,686,448,691,811,812,724,501,57,0,57,415,610,700,706,558,0,178],[0,0,0,538,788,881,902,865,772,734,790,817,786,682,462,90,0,23,329,521,606,585,296,0,0],[370,84,318,668,831,892,902,878,844,831,831,813,755,630,395,32,0,0,209,408,493,446,32,0,104],[617,493,592,773,862,886,891,893,885,869,845,800,717,565,294,0,0,0,37,266,365,312,0,0,354],[763,701,756,854,877,839,870,903,903,881,842,778,671,483,151,0,0,0,0,89,229,214,10,0,535],[852,817,856,916,868,650,848,906,905,878,829,749,616,383,0,0,0,0,0,0,101,169,156,293,651],[905,879,913,960,826,0,835,897,894,864,807,713,554,264,0,0,0,0,0,0,0,172,292,481,726],[933,903,931,984,805,314,819,877,874,841,778,672,486,131,0,0,0,0,0,0,0,209,400,592,777],[936,874,885,980,779,486,780,843,844,812,745,628,418,0,0,0,0,0,0,0,0,265,485,665,814],[904,680,724,935,569,305,705,796,807,778,710,585,356,0,0,0,0,0,0,0,0,329,553,716,842],[782,0,829,853,0,0,593,738,766,743,676,549,311,0,0,0,0,0,0,0,43,396,608,755,864],[130,390,879,783,0,0,472,679,726,711,648,524,291,0,0,0,0,0,0,0,142,462,654,786,882],[0,707,889,766,0,0,408,635,695,687,630,513,299,0,0,0,0,0,0,0,248,524,695,812,897]],"best":[3.23585,2.19754],"mean":[[961,952,939,923,902,877,847,811,767,713,646,563,461,367,357,435,517,570,592,587,558,514,468,449,470],[960,951,937,920,898,872,840,802,755,697,624,529,404,279,283,403,505,564,586,576,538,477,408,372,404],[958,948,934,916,893,865,832,791,742,680,600,492,341,169,213,382,499,562,583,569,522,442,340,275,324],[956,945,930,911,887,857,822,780,728,662,576,456,275,50,174,379,501,564,583,565,511,415,273,154,236],[952,940,925,904,879,848,811,766,712,644,553,424,224,0,196,396,512,571,588,568,510,405,234,46,163],[946,934,917,896,869,836,798,752,696,625,533,403,218,97,267,430,531,584,598,576,519,415,249,58,155],[939,926,908,885,857,823,783,735,679,608,518,401,267,235,354,476,558,602,613,592,538,445,308,173,214],[931,917,897,873,843,807,765,717,661,594,513,420,346,355,439,526,590,626,633,614,565,487,382,289,296],[920,905,884,857,825,788,746,698,644,583,518,457,427,452,514,578,627,654,659,640,598,534,453,384,375],[907,890,867,839,805,766,723,677,628,578,533,502,501,532,580,628,665,686,688,670,634,580,517,462,445],[891,872,847,816,780,740,697,654,614,580,556,550,565,598,638,675,703,718,718,702,671,625,573,527,506],[872,851,823,789,751,709,669,632,604,587,585,597,622,654,689,719,740,751,749,734,706,667,623,582,559],[850,825,794,757,716,674,637,611,598,601,616,641,671,703,733,758,775,783,780,766,740,706,667,631,607],[824,795,759,718,674,633,604,592,599,620,649,681,714,745,772,794,808,813,809,795,772,742,707,674,649],[794,760,718,671,625,588,571,580,606,643,681,718,752,782,807,826,838,841,837,823,802,774,742,712,687],[760,718,669,616,567,538,543,575,620,668,712,751,785,814,837,855,865,867,862,849,829,804,774,746,721],[722,671,611,549,501,490,524,580,640,694,741,781,815,843,865,880,889,891,885,873,854,830,803,776,752],[680,617,543,470,430,453,520,596,663,720,768,808,841,868,888,903,911,912,906,894,876,854,829,803,780],[636,557,465,383,369,439,534,619,689,746,793,832,864,890,909,923,930,931,925,913,896,875,851,827,805],[592,496,383,304,346,456,561,647,716,771,816,854,884,909,928,940,947,947,942,930,914,894,872,848,826],[554,443,319,278,373,495,597,677,742,794,837,873,902,926,943,955,962,962,956,945,930,911,889,867,846],[529,415,311,324,433,544,635,708,767,816,856,890,918,940,957,968,974,974,969,958,943,925,905,884,863],[521,423,361,402,499,593,672,737,791,836,873,905,931,952,968,979,985,984,979,969,955,938,918,898,878],[533,460,434,480,560,639,706,763,812,853,889,918,943,963,978,988,993,993,988,978,965,948,930,911,892],[557,510,504,547,613,678,737,787,831,869,902,930,953,972,986,995,1000,1000,995,986,973,958,940,922,903]],"newest":[8.30079,0.0],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0],[0.890363,6.32204],[-5.0,15.0],[3.23585,2.19754],[9.09776,2.7362],[9.79007,2.3748],[8.30079,0.0]]}}, +{"best":0.0426524,"evaluations":20,"generation":14,"state":{"acquisition":[[733,727,740,765,791,810,814,797,741,597,227,350,890,997,952,856,826,786,759,734,587,0,362,578,0],[664,656,678,717,756,786,798,784,727,555,0,0,833,990,778,566,569,441,461,586,620,427,741,804,0],[574,564,600,660,720,765,788,784,739,601,31,0,829,991,0,568,146,0,0,475,724,814,915,934,465],[454,439,499,595,684,748,785,793,768,690,528,555,926,994,939,786,94,0,0,464,779,885,945,984,700],[287,260,366,519,649,737,788,808,800,769,751,855,991,983,981,835,255,0,0,527,799,887,873,981,800],[42,0,181,432,617,732,796,825,827,818,840,930,1000,0,941,784,248,0,19,603,807,874,831,978,753],[0,0,0,335,592,734,808,839,842,833,858,936,968,629,805,581,0,0,302,659,813,873,890,955,0],[0,0,0,237,578,742,821,850,841,807,810,896,919,767,549,0,0,0,409,688,816,875,905,918,368],[0,0,0,152,579,758,836,856,821,704,621,820,888,793,383,0,0,0,441,695,810,866,891,868,680],[0,0,0,108,597,779,851,857,774,362,0,763,869,790,397,0,0,0,438,681,791,842,851,785,697],[0,0,0,131,632,806,867,855,698,0,0,765,852,772,442,0,0,0,417,650,756,797,776,616,611],[0,0,0,225,681,835,883,851,627,0,368,790,832,746,462,0,0,0,381,602,702,727,648,200,403],[0,0,0,367,735,862,895,851,650,379,673,809,810,710,451,0,0,0,324,534,627,627,440,0,62],[0,0,0,519,788,884,903,857,748,712,783,816,782,665,410,0,0,0,238,443,531,497,143,0,0],[321,0,254,656,833,896,901,870,832,823,828,812,748,608,338,0,0,0,111,325,412,347,0,0,16],[592,441,557,769,867,888,887,888,881,867,843,797,707,538,228,0,0,0,0,172,274,205,0,0,300],[751,676,740,857,883,833,862,901,902,881,840,773,657,449,72,0,0,0,0,0,128,101,0,0,502],[849,807,851,924,873,612,839,905,905,877,826,741,598,341,0,0,0,0,0,0,0,55,35,205,631],[908,877,915,972,828,0,828,897,895,862,801,701,530,212,0,0,0,0,0,0,0,65,200,426,714],[939,904,936,999,799,243,812,875,872,837,770,656,455,68,0,0,0,0,0,0,0,113,331,555,771],[943,874,886,994,764,432,769,838,839,804,733,607,379,0,0,0,0,0,0,0,0,182,432,639,812],[909,663,715,947,522,230,686,785,798,767,693,559,310,0,0,0,0,0,0,0,0,260,512,699,842],[777,0,836,860,0,0,561,721,752,727,655,516,257,0,0,0,0,0,0,0,0,339,577,743,866],[64,343,889,784,0,0,424,654,706,690,622,485,230,0,0,0,0,0,0,0,59,415,631,778,886],[0,705,899,764,0,0,346,602,669,661,599,470,233,0,0,0,0,0,0,0,179,486,677,808,903]],"best":[3.23585,2.19754],"mean":[[970,960,947,930,909,883,853,816,771,716,648,561,455,355,343,422,501,549,563,546,502,444,400,412,470],[968,958,944,926,904,877,845,806,759,700,625,527,398,268,276,397,496,550,565,544,491,411,335,330,405],[965,954,940,921,898,870,836,795,745,683,601,491,334,159,215,384,497,555,570,547,487,388,268,228,327],[961,950,934,915,890,861,825,783,730,664,577,455,269,42,185,387,504,563,578,554,490,381,220,109,242],[956,944,928,907,881,850,813,769,715,646,554,423,219,0,209,405,518,575,588,565,503,393,217,31,171],[950,937,920,898,870,838,800,754,698,627,534,403,215,98,275,438,537,589,602,580,522,421,264,85,160],[942,928,910,886,858,824,784,737,680,610,520,401,264,233,356,480,563,608,619,599,548,461,334,202,214],[932,917,897,872,842,807,766,718,662,595,514,419,341,350,436,527,593,631,640,623,578,505,406,309,294],[921,904,883,856,824,787,745,698,645,584,518,454,421,445,509,576,628,658,665,649,611,550,472,398,371],[907,889,865,836,802,764,722,676,628,579,532,498,494,524,574,624,664,687,692,678,644,594,531,470,440],[890,870,844,813,777,737,695,653,614,579,554,545,558,590,631,670,701,718,721,708,678,635,583,532,501],[871,848,819,785,747,706,666,630,603,586,582,592,614,646,681,713,737,750,750,738,712,674,629,585,554],[848,822,790,752,711,670,634,609,597,599,613,635,664,695,726,752,771,781,780,767,744,711,671,632,602],[822,792,755,713,669,629,601,590,597,617,645,675,707,738,765,787,803,810,808,796,774,744,709,673,644],[792,756,713,666,619,583,568,578,605,640,676,712,745,775,800,820,833,838,834,823,802,775,743,711,683],[758,715,664,610,561,534,540,573,618,665,707,746,779,808,831,849,860,863,859,848,829,804,774,744,718],[721,668,607,543,495,487,522,579,638,691,737,776,809,837,859,875,884,887,882,871,853,830,802,774,749],[679,614,539,465,426,451,519,595,661,718,764,803,836,862,883,898,906,908,903,892,875,853,828,801,777],[636,555,461,378,366,438,533,618,688,744,790,828,860,885,904,918,926,927,922,911,895,874,850,826,802],[593,495,380,301,344,456,561,647,714,769,814,851,881,905,923,936,943,944,939,928,913,893,871,847,825],[556,443,318,276,372,495,597,677,741,793,836,871,899,922,940,952,958,959,954,944,929,910,889,866,845],[530,415,310,321,432,544,635,708,767,815,855,888,916,938,954,966,972,972,967,957,943,925,905,884,863],[522,423,359,399,498,593,672,737,791,836,873,904,930,951,967,977,983,983,978,968,955,938,919,899,878],[532,458,430,477,559,638,706,764,813,854,889,918,943,962,977,987,992,992,987,978,965,949,931,912,892],[556,507,500,544,611,678,737,789,833,871,903,931,953,972,986,995,1000,1000,995,987,974,959,942,923,905]],"newest":[2.86339,2.49195],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0],[0.890363,6.32204],[-5.0,15.0],[3.23585,2.19754],[9.09776,2.7362],[9.79007,2.3748],[8.30079,0.0],[2.86339,2.49195]]}}, +{"best":0.027846,"evaluations":21,"generation":15,"state":{"acquisition":[[662,647,656,678,702,717,711,672,567,283,0,123,873,994,929,792,763,718,689,664,489,0,243,511,0],[581,563,580,619,661,690,696,667,572,286,0,0,791,984,688,431,449,278,311,484,537,313,690,773,0],[476,452,486,552,620,669,692,681,618,437,0,0,636,982,0,454,0,0,0,354,666,778,899,926,354],[335,301,364,475,580,656,699,708,680,601,434,104,286,979,842,664,0,0,0,346,733,863,937,987,653],[138,86,204,388,545,652,714,740,738,715,691,687,599,972,882,687,0,0,0,421,755,865,856,989,774],[0,0,0,291,517,656,734,771,780,778,788,823,834,0,740,566,0,0,0,508,761,847,801,982,707],[0,0,0,188,499,667,756,795,804,797,804,840,858,520,349,164,0,0,116,570,765,841,865,952,0],[0,0,0,89,495,687,778,813,807,767,736,791,855,714,45,0,0,0,241,601,765,840,882,905,211],[0,0,0,14,508,712,801,824,785,649,472,715,850,750,127,0,0,0,272,606,756,830,864,843,601],[0,0,0,0,540,743,823,827,731,251,0,689,840,749,257,0,0,0,261,588,733,800,816,740,621],[0,0,0,34,588,778,843,825,638,0,0,719,824,730,333,0,0,0,232,550,691,747,729,536,514],[0,0,0,150,647,812,861,818,536,0,263,755,803,701,361,0,0,0,190,492,627,665,579,38,267],[0,0,0,308,710,844,875,813,546,216,616,778,778,661,351,0,0,0,127,414,541,549,337,0,0],[0,0,0,474,769,869,881,814,670,643,744,786,747,611,306,0,0,0,32,309,429,398,0,0,0],[226,0,145,623,819,881,876,828,781,781,795,780,710,548,226,0,0,0,0,173,291,223,0,0,0],[533,343,488,745,856,870,855,850,845,835,813,763,663,469,104,0,0,0,0,0,132,54,0,0,195],[714,620,697,842,874,804,823,870,874,853,810,736,607,370,0,0,0,0,0,0,0,0,0,0,425],[826,772,826,917,863,529,798,878,880,850,793,699,539,249,0,0,0,0,0,0,0,0,0,47,572],[893,855,899,970,810,0,791,871,869,832,765,654,462,104,0,0,0,0,0,0,0,0,42,318,668],[928,887,924,1000,767,112,776,847,843,804,728,601,376,0,0,0,0,0,0,0,0,0,205,475,734],[934,855,866,996,719,333,727,805,806,766,685,543,286,0,0,0,0,0,0,0,0,38,331,577,782],[896,618,678,943,427,98,631,744,758,721,638,485,203,0,0,0,0,0,0,0,0,135,429,649,817],[751,0,823,847,0,0,486,668,703,674,591,433,137,0,0,0,0,0,0,0,0,233,508,702,845],[0,248,882,764,0,0,322,588,648,629,550,393,99,0,0,0,0,0,0,0,0,325,573,744,868],[0,680,893,739,0,0,222,523,601,593,521,372,97,0,0,0,0,0,0,0,60,409,628,778,887]],"best":[-3.15455,12.1417],"mean":[[981,970,956,938,917,890,859,821,775,719,649,561,453,351,340,419,499,548,562,545,502,443,399,410,468],[978,966,952,933,910,883,850,810,762,702,626,526,395,263,274,396,494,549,563,543,490,410,333,328,403],[974,962,946,927,903,874,840,798,748,684,601,489,330,154,215,384,496,554,568,545,486,388,267,226,326],[968,956,940,919,894,864,828,785,732,665,577,452,263,39,189,388,504,562,576,552,489,381,220,108,242],[962,949,932,910,884,853,815,770,716,646,553,420,213,0,214,407,518,573,587,563,502,393,217,32,173],[955,940,922,899,872,839,800,754,698,627,532,399,209,99,279,440,537,588,601,579,521,421,264,85,161],[945,930,911,887,858,824,784,736,680,609,518,397,260,232,358,481,563,607,618,598,547,460,333,201,214],[935,918,897,872,841,806,765,717,661,594,512,416,337,348,436,527,593,630,639,621,576,503,404,308,293],[922,904,882,854,822,785,743,696,643,583,516,451,417,442,507,575,626,656,663,647,609,548,470,396,370],[907,887,863,833,799,761,719,674,627,577,530,495,490,520,571,622,662,685,690,675,642,591,528,468,439],[890,868,841,809,773,733,692,651,612,578,552,542,554,586,627,667,698,716,718,705,676,633,580,530,499],[870,845,815,780,742,701,662,628,601,584,580,588,610,642,677,709,733,746,747,734,709,671,627,583,553],[847,818,785,746,705,665,630,606,595,597,610,632,659,690,721,747,767,776,776,764,740,707,668,630,600],[820,788,749,706,662,623,597,588,596,616,642,672,702,733,760,782,798,805,803,792,770,741,706,671,643],[790,752,707,659,612,578,565,576,603,638,674,709,741,770,795,814,827,833,830,818,799,772,741,709,682],[756,711,658,603,554,529,538,572,618,663,705,742,775,803,826,843,854,858,855,844,825,800,772,742,717],[719,664,600,536,489,483,521,579,637,690,734,773,805,832,853,869,879,882,878,867,849,827,800,773,748],[678,611,533,458,420,449,519,595,661,716,762,800,832,858,878,893,901,903,899,888,872,850,826,800,777],[635,553,456,372,363,438,534,618,687,743,788,826,856,881,900,913,921,923,918,908,892,872,849,825,802],[594,494,377,297,342,456,562,647,714,768,812,848,878,901,919,932,939,940,935,925,910,892,870,847,825],[557,443,316,273,371,495,597,677,741,793,834,869,897,919,936,948,955,955,951,941,927,909,889,867,846],[532,416,308,318,430,543,635,708,767,815,855,887,914,935,952,963,968,969,964,955,942,925,905,885,865],[523,422,356,396,495,592,672,738,792,836,873,904,929,949,965,975,981,981,976,967,955,939,920,901,881],[532,456,427,474,556,637,707,765,814,855,890,919,943,962,976,986,991,991,987,978,966,951,933,915,896],[554,504,497,542,610,678,738,790,835,873,905,932,954,972,986,995,1000,1000,996,988,976,962,945,927,910]],"newest":[-3.15455,12.1417],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0],[0.890363,6.32204],[-5.0,15.0],[3.23585,2.19754],[9.09776,2.7362],[9.79007,2.3748],[8.30079,0.0],[2.86339,2.49195],[-3.15455,12.1417]]}}, +{"best":0.027846,"evaluations":22,"generation":16,"state":{"acquisition":[[613,605,623,656,689,711,711,677,578,303,0,94,852,973,907,772,747,703,677,658,496,0,281,531,0],[526,517,548,601,653,689,701,678,590,321,0,0,771,964,669,430,449,273,311,490,546,316,681,766,0],[413,402,454,538,618,675,703,696,638,468,0,0,618,961,0,475,0,0,0,375,669,769,881,910,334],[262,246,335,468,585,667,713,725,699,623,460,140,281,955,839,676,0,0,0,375,735,854,919,969,645],[51,23,180,390,558,668,731,757,755,731,706,701,613,946,878,698,0,0,0,447,758,859,845,972,765],[0,0,0,305,538,676,752,788,796,792,800,831,835,0,742,581,0,0,0,525,764,843,794,966,687],[0,0,0,219,528,691,775,812,819,811,816,849,855,496,362,191,0,0,139,581,765,835,851,937,0],[0,0,0,139,531,712,798,830,823,784,752,801,849,695,50,0,0,0,260,608,763,831,866,889,164],[0,0,0,83,549,738,819,839,802,672,496,725,840,733,123,0,0,0,292,610,752,819,848,825,565],[0,0,0,70,583,768,838,840,749,291,0,693,826,731,246,0,0,0,282,592,729,790,801,720,586],[0,0,0,117,630,799,853,830,650,0,0,707,805,711,317,0,0,0,253,556,688,739,715,512,480],[0,0,0,222,685,829,864,811,514,0,206,728,779,678,340,0,0,0,210,501,629,662,572,11,239],[0,0,0,364,744,856,866,778,437,0,539,738,748,635,325,0,0,0,146,426,547,553,343,0,0],[0,0,0,514,799,874,854,733,490,473,658,734,709,579,276,0,0,0,51,327,443,412,19,0,0],[185,0,122,651,846,879,817,681,583,632,703,718,663,510,190,0,0,0,0,198,314,250,0,0,0],[493,350,480,765,882,857,733,634,647,692,713,691,608,424,63,0,0,0,0,33,167,93,0,0,195],[674,624,699,856,901,767,567,607,677,708,703,654,542,318,0,0,0,0,0,0,9,0,0,0,419],[781,772,834,924,886,363,350,598,683,701,678,608,466,189,0,0,0,0,0,0,0,0,0,37,564],[843,848,911,970,736,0,288,592,670,679,643,554,380,38,0,0,0,0,0,0,0,0,55,311,660],[871,871,934,973,421,0,342,576,645,647,601,495,287,0,0,0,0,0,0,0,0,0,214,470,726],[865,819,815,1000,651,0,350,544,609,608,554,433,194,0,0,0,0,0,0,0,0,68,338,573,774],[810,490,528,964,357,0,278,494,566,566,508,375,111,0,0,0,0,0,0,0,0,160,435,645,810],[624,0,855,882,0,0,146,434,522,526,467,327,48,0,0,0,0,0,0,0,0,253,513,699,838],[0,147,897,804,0,0,18,380,485,494,435,295,18,0,0,0,0,0,0,0,0,340,576,741,861],[0,693,906,772,0,0,0,352,462,474,419,285,25,0,0,0,0,0,0,0,86,421,630,775,880]],"best":[-3.15455,12.1417],"mean":[[988,976,960,942,919,891,859,821,775,719,650,563,457,357,344,422,501,550,564,547,503,443,398,409,468],[984,972,955,936,912,883,849,809,760,700,625,527,399,269,276,397,495,550,564,544,491,410,332,327,403],[980,966,949,929,903,873,838,795,744,681,599,488,332,159,214,383,495,554,569,546,486,388,267,225,327],[974,960,942,920,894,862,825,781,727,660,571,448,263,41,185,386,503,562,576,553,490,381,220,107,242],[968,953,934,910,882,849,811,765,709,638,545,413,209,0,211,405,516,573,587,564,502,393,218,31,173],[960,944,923,899,869,835,795,747,689,617,522,389,202,98,277,438,536,588,601,579,522,422,264,84,161],[950,933,911,885,854,818,777,728,670,598,505,385,253,232,358,480,562,607,618,599,547,460,333,200,214],[939,920,897,869,836,799,757,708,651,582,499,405,333,348,436,527,592,630,639,622,577,504,405,307,293],[926,905,880,850,815,777,735,687,634,573,506,444,415,443,508,575,626,656,664,648,609,549,470,396,370],[911,888,860,827,791,752,711,667,619,570,524,493,490,522,572,622,662,685,690,676,643,592,529,469,440],[894,868,837,801,763,724,685,646,609,576,552,544,557,588,628,667,698,716,718,705,676,633,580,530,501],[873,844,809,770,730,692,657,628,605,590,585,594,614,645,678,709,733,746,747,734,709,671,627,583,554],[850,816,776,733,691,655,630,614,608,610,621,640,665,694,722,748,766,776,775,763,740,707,668,630,601],[824,784,737,688,645,615,603,605,617,635,657,682,709,737,762,783,798,805,803,791,770,740,706,671,644],[794,747,691,634,590,572,580,603,632,662,692,721,749,774,797,815,827,832,829,817,798,771,739,708,682],[762,704,636,567,524,528,563,608,651,690,725,755,783,808,828,844,854,857,854,842,823,799,770,741,716],[726,656,569,482,447,489,557,621,674,718,755,787,814,837,856,870,879,881,876,865,847,825,798,771,747],[689,604,489,370,363,463,562,639,698,745,783,814,841,863,881,894,901,902,897,886,870,848,823,798,775],[651,548,395,220,289,458,579,662,723,771,808,839,865,886,903,915,921,921,916,906,890,869,846,822,800],[614,495,297,42,269,478,603,687,748,794,832,862,887,907,922,933,939,939,934,923,908,889,867,844,823],[583,454,235,38,321,514,633,713,772,817,853,882,906,925,940,949,954,954,949,939,924,906,885,864,843],[559,433,255,206,402,559,664,739,794,837,872,900,922,941,955,964,968,968,963,953,939,921,902,881,861],[546,438,330,346,481,604,696,763,815,856,889,915,937,955,968,976,981,980,975,965,952,935,917,897,878],[545,464,411,449,549,647,725,786,834,873,904,929,950,967,979,987,991,990,985,976,963,948,930,911,893],[554,500,482,525,605,685,753,808,852,888,917,942,962,977,989,997,1000,999,994,985,973,958,942,924,906]],"newest":[-3.15626,12.5625],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0],[0.890363,6.32204],[-5.0,15.0],[3.23585,2.19754],[9.09776,2.7362],[9.79007,2.3748],[8.30079,0.0],[2.86339,2.49195],[-3.15455,12.1417],[-3.15626,12.5625]]}}, +{"best":0.027846,"evaluations":23,"generation":17,"state":{"acquisition":[[442,434,478,546,611,654,666,633,520,194,0,0,837,981,899,718,691,633,605,597,425,0,246,527,0],[315,305,374,478,572,636,662,642,544,233,0,0,747,974,624,320,341,91,156,413,497,202,636,760,0],[148,133,246,404,539,628,672,669,604,404,0,0,561,973,0,430,0,0,0,305,642,744,869,917,169],[0,0,84,326,513,631,693,708,676,579,373,0,140,973,844,661,0,0,0,322,721,850,918,988,612],[0,0,0,245,497,643,720,749,740,702,662,664,583,967,889,681,0,0,0,402,750,862,845,1000,760],[0,0,0,167,492,665,752,787,788,771,772,814,823,0,732,535,0,0,0,480,753,844,785,987,629],[0,0,0,100,502,692,783,818,816,792,787,828,832,358,267,49,0,0,0,530,748,825,831,948,0],[0,0,0,55,525,725,813,841,822,760,702,762,819,623,0,0,0,0,122,553,737,811,842,884,0],[0,0,0,42,562,760,840,854,803,629,368,667,810,676,0,0,0,0,167,552,718,789,817,795,395],[0,0,0,69,608,795,862,857,748,188,0,635,796,678,80,0,0,0,162,529,687,753,758,651,423],[0,0,0,134,660,829,879,847,641,0,0,660,775,658,170,0,0,0,129,487,640,694,657,376,288],[0,0,0,227,714,859,888,823,477,0,56,687,746,623,199,0,0,0,75,424,573,608,494,0,10],[0,0,0,334,766,882,886,778,349,0,450,698,711,574,182,0,0,0,0,339,484,489,240,0,0],[0,0,0,439,812,896,866,712,380,344,593,694,668,512,125,0,0,0,0,227,369,337,0,0,0],[0,0,0,534,851,894,815,632,487,549,648,676,616,434,26,0,0,0,0,81,228,159,0,0,0],[139,0,0,614,878,862,704,564,569,626,661,645,554,337,0,0,0,0,0,0,64,0,0,0,68],[392,0,0,677,888,746,490,530,609,648,650,603,480,220,0,0,0,0,0,0,0,0,0,0,314],[556,289,4,724,860,221,231,520,616,640,620,549,395,78,0,0,0,0,0,0,0,0,0,0,482],[658,492,326,753,677,0,156,506,597,612,578,486,297,0,0,0,0,0,0,0,0,0,0,121,596],[713,580,602,783,297,0,189,471,557,567,523,413,190,0,0,0,0,0,0,0,0,0,26,337,677],[721,523,607,931,337,0,153,405,497,510,460,335,79,0,0,0,0,0,0,0,0,0,188,476,736],[661,0,72,612,0,0,0,304,421,443,392,255,0,0,0,0,0,0,0,0,0,5,316,572,781],[427,0,792,693,0,0,0,169,333,371,324,181,0,0,0,0,0,0,0,0,0,118,417,642,816],[0,2,897,731,0,0,0,19,242,301,262,119,0,0,0,0,0,0,0,0,0,225,499,696,844],[0,696,927,763,0,0,0,0,165,244,214,79,0,0,0,0,0,0,0,0,0,322,567,740,868]],"best":[-3.15455,12.1417],"mean":[[999,985,968,947,923,894,861,822,776,720,650,563,458,358,345,423,503,551,565,548,504,442,394,404,467],[994,979,961,939,914,884,849,808,759,699,624,525,398,269,276,396,495,549,565,544,491,410,328,321,402],[988,972,953,930,903,872,836,793,742,678,596,485,330,159,213,381,493,552,568,546,486,388,263,219,326],[981,964,944,920,892,859,821,777,723,656,568,445,261,41,183,383,500,560,575,552,490,382,219,102,243],[973,955,934,908,878,844,805,759,703,633,541,410,207,0,208,402,514,571,586,563,503,395,219,27,175],[963,944,922,895,863,828,787,740,683,612,518,387,202,98,275,435,534,587,600,579,523,423,266,80,163],[953,932,908,879,846,809,767,719,662,592,502,384,253,231,355,477,560,606,618,599,549,462,334,196,213],[941,919,892,861,826,788,746,698,643,577,497,405,332,345,433,524,591,630,640,623,578,505,404,304,291],[927,903,874,841,804,764,722,676,625,567,504,444,414,439,504,572,625,656,664,649,610,549,469,393,369],[912,885,853,817,778,738,697,654,611,566,523,492,488,517,567,619,660,684,690,676,643,592,527,466,438],[894,864,829,790,749,708,670,634,602,573,551,542,553,583,623,663,695,714,718,705,676,632,579,528,499],[874,841,801,758,714,675,642,617,599,587,584,591,610,639,672,704,729,744,745,733,707,670,625,581,553],[852,814,769,720,674,638,614,603,602,608,620,637,660,688,716,742,762,773,773,761,738,705,666,627,600],[828,783,731,676,628,597,588,596,613,634,656,679,704,730,755,777,793,801,799,788,767,737,703,668,642],[800,749,687,623,573,554,568,597,630,661,690,718,743,768,790,809,821,827,825,814,794,767,736,705,680],[771,710,635,558,508,513,555,605,651,690,723,752,778,801,821,837,848,852,849,838,819,795,767,738,714],[738,666,574,476,433,477,552,620,675,718,754,784,809,831,850,864,872,875,871,860,843,821,794,768,745],[705,618,501,371,352,456,561,641,700,746,782,812,837,858,875,887,895,896,892,881,865,844,820,795,773],[670,568,416,230,283,457,581,665,726,772,808,838,862,882,897,908,915,916,911,901,885,865,843,819,798],[635,518,324,55,265,480,607,691,752,797,832,861,884,903,917,928,933,933,929,918,904,885,864,842,821],[604,475,251,3,315,518,638,718,776,820,854,882,904,922,935,945,950,950,944,935,920,903,883,862,842],[577,446,242,162,395,562,670,745,799,841,874,900,922,939,952,960,965,964,959,949,936,919,900,880,861],[559,438,299,311,473,608,702,770,821,861,892,917,938,954,966,974,978,977,972,963,950,934,915,896,878],[550,452,378,421,542,650,732,794,841,879,908,933,952,967,979,986,990,989,984,975,962,947,930,911,894],[552,481,452,504,599,688,760,816,860,895,923,946,965,979,990,997,1000,999,994,985,973,959,942,925,908]],"newest":[9.49298,2.83877],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0],[0.890363,6.32204],[-5.0,15.0],[3.23585,2.19754],[9.09776,2.7362],[9.79007,2.3748],[8.30079,0.0],[2.86339,2.49195],[-3.15455,12.1417],[-3.15626,12.5625],[9.49298,2.83877]]}}, +{"best":0.00131003,"evaluations":24,"generation":18,"state":{"acquisition":[[345,335,389,473,551,602,618,582,455,83,0,0,815,977,886,672,634,563,538,537,340,0,126,430,0],[199,186,272,398,511,586,617,595,485,133,0,0,715,971,579,204,216,0,25,311,341,0,557,740,0],[6,0,127,318,477,581,632,629,556,330,0,0,503,974,0,356,0,0,0,111,438,395,761,913,0],[0,0,0,234,453,588,658,675,638,529,294,0,17,979,834,623,0,0,0,52,508,579,722,994,0],[0,0,0,151,441,607,692,723,711,667,621,625,541,977,883,644,0,0,0,138,552,632,233,1000,369],[0,0,0,75,444,635,730,767,766,744,743,792,804,0,705,474,0,0,0,267,590,682,647,863,437],[0,0,0,13,461,669,767,804,798,768,759,804,809,254,165,0,0,0,0,369,625,730,801,871,0],[0,0,0,0,493,708,802,830,807,733,659,725,791,566,0,0,0,0,0,428,646,753,820,810,0],[0,0,0,0,537,749,833,846,787,587,269,613,780,627,0,0,0,0,3,449,647,746,790,713,285],[0,0,0,15,590,789,858,849,728,93,0,581,765,632,0,0,0,0,16,439,626,713,723,546,315],[0,0,0,91,648,826,876,839,608,0,0,613,743,610,49,0,0,0,0,403,582,652,608,218,148],[0,0,0,192,707,858,885,810,420,0,0,645,712,572,86,0,0,0,0,341,514,558,425,0,0],[0,0,0,304,762,882,882,758,258,0,376,659,673,518,70,0,0,0,0,252,418,429,143,0,0],[0,0,0,414,812,897,858,678,280,242,538,654,626,450,8,0,0,0,0,131,294,262,0,0,0],[0,0,0,512,852,894,799,579,404,481,600,635,569,365,0,0,0,0,0,0,139,64,0,0,0],[40,0,0,595,881,857,669,496,501,571,615,601,501,260,0,0,0,0,0,0,0,0,0,0,0],[320,0,0,661,892,727,418,458,550,598,603,554,420,131,0,0,0,0,0,0,0,0,0,0,181],[501,189,0,711,860,126,117,449,560,590,571,495,327,0,0,0,0,0,0,0,0,0,0,0,385],[614,421,223,745,660,0,38,435,540,558,523,425,220,0,0,0,0,0,0,0,0,0,0,0,522],[676,522,543,784,205,0,76,397,495,509,463,344,101,0,0,0,0,0,0,0,0,0,0,228,619],[686,459,547,938,238,0,33,322,428,444,392,257,0,0,0,0,0,0,0,0,0,0,67,392,689],[621,0,0,588,0,0,0,207,341,368,315,166,0,0,0,0,0,0,0,0,0,0,216,505,742],[366,0,790,677,0,0,0,52,240,286,236,81,0,0,0,0,0,0,0,0,0,1,335,588,784],[0,0,902,719,0,0,0,0,135,205,164,8,0,0,0,0,0,0,0,0,0,125,430,651,817],[0,690,933,752,0,0,0,0,44,137,107,0,0,0,0,0,0,0,0,0,0,237,508,702,845]],"best":[-3.1535,12.3287],"mean":[[1000,985,967,946,921,892,858,819,772,716,647,560,455,357,343,419,498,546,561,545,501,439,389,397,463],[994,978,959,937,911,880,845,804,755,695,620,522,395,268,274,392,489,544,560,542,490,410,326,316,400],[987,971,951,927,900,868,831,788,737,673,591,481,328,158,212,377,489,547,564,544,487,391,267,215,327],[979,962,941,916,887,854,816,771,717,650,563,441,258,41,181,379,495,555,572,551,492,388,226,99,247],[971,952,929,903,873,838,799,753,697,628,536,406,205,0,206,398,509,567,583,562,505,400,226,18,181],[961,941,917,889,857,821,780,733,676,606,513,383,200,97,272,431,530,583,598,578,524,427,269,66,165],[950,928,902,872,839,801,760,712,656,586,497,381,252,229,352,474,557,603,616,598,549,464,333,183,212],[937,913,886,854,818,780,737,690,636,571,492,402,330,342,429,520,587,626,637,621,578,505,401,294,288],[923,897,867,833,795,755,713,668,618,562,500,441,410,435,499,567,620,652,661,646,608,547,464,384,365],[907,879,846,808,769,728,687,646,604,561,519,489,484,512,562,613,655,680,686,673,640,588,522,459,435],[889,858,821,780,738,698,660,626,595,568,547,538,548,577,617,657,689,709,713,700,671,627,573,521,496],[869,834,793,748,704,664,632,609,593,583,580,587,605,633,666,698,723,738,740,728,702,664,618,575,549],[847,807,760,710,663,627,605,596,597,604,615,632,654,681,709,735,755,766,767,755,732,699,659,621,597],[823,777,723,666,616,586,580,590,608,629,651,674,698,723,748,769,785,794,793,781,760,731,696,663,639],[796,742,679,612,561,544,560,591,625,657,685,712,737,761,782,801,814,820,817,807,787,761,730,700,677],[767,704,627,547,496,503,548,600,646,685,718,746,771,794,813,829,840,844,841,830,812,788,761,733,711],[735,661,567,466,422,469,547,616,671,714,748,777,802,823,841,855,864,867,863,853,836,814,788,763,741],[702,615,495,362,342,451,557,637,696,741,777,806,830,850,867,879,886,888,884,874,858,837,814,790,769],[668,566,412,223,277,453,577,661,722,767,803,831,855,874,889,900,907,908,903,893,878,859,837,815,795],[634,517,323,52,264,477,604,688,747,792,827,854,877,895,910,920,925,925,921,911,897,879,858,837,818],[603,474,251,3,314,515,635,715,772,815,849,875,897,915,928,937,942,942,937,927,914,897,877,858,839],[576,445,239,157,392,559,667,741,795,836,869,895,915,932,944,953,957,957,952,942,929,913,895,876,858],[557,435,292,305,469,604,698,766,817,856,887,912,932,947,959,967,971,970,965,956,944,928,911,893,876],[547,447,369,414,537,646,728,790,837,874,904,927,946,961,972,980,983,982,977,969,957,942,926,908,892],[548,474,443,496,593,684,756,812,856,891,919,942,960,974,984,991,994,993,988,980,968,955,939,922,906]],"newest":[-3.1535,12.3287],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0],[0.890363,6.32204],[-5.0,15.0],[3.23585,2.19754],[9.09776,2.7362],[9.79007,2.3748],[8.30079,0.0],[2.86339,2.49195],[-3.15455,12.1417],[-3.15626,12.5625],[9.49298,2.83877],[-3.1535,12.3287]]}}, +{"best":0.00131003,"evaluations":25,"generation":19,"state":{"acquisition":[[0,0,0,242,434,547,591,561,414,0,0,0,664,910,776,438,414,269,256,344,174,0,87,402,0],[0,0,0,183,421,559,617,599,472,53,0,0,531,912,366,0,0,0,0,136,250,0,447,705,0],[0,0,0,123,417,581,653,650,554,255,0,0,226,924,0,291,0,0,0,6,391,259,644,895,0],[0,0,0,67,423,611,696,706,642,459,49,0,0,945,812,580,0,0,0,11,488,537,562,993,0],[0,0,0,14,439,648,742,762,718,606,475,499,466,952,874,587,0,0,0,98,537,621,147,999,0],[0,0,0,0,465,691,789,812,776,691,632,710,734,0,665,351,0,0,0,190,556,650,546,774,0],[0,0,0,0,500,736,832,853,812,713,641,712,688,0,0,0,0,0,0,255,559,658,688,777,0],[0,0,0,0,543,781,871,883,822,661,480,583,619,215,0,0,0,0,0,285,548,649,693,658,0],[0,0,0,0,591,825,904,900,802,460,0,398,586,336,0,0,0,0,0,285,522,619,640,469,0],[0,0,0,0,644,866,929,903,736,0,0,315,562,359,0,0,0,0,0,257,480,565,541,159,0],[0,0,0,0,698,902,945,889,599,0,0,335,531,341,0,0,0,0,0,202,417,484,387,0,0],[0,0,0,0,751,931,949,848,366,0,0,370,492,296,0,0,0,0,0,121,332,372,160,0,0],[0,0,0,56,803,951,935,773,119,0,0,388,443,231,0,0,0,0,0,10,221,226,0,0,0],[0,0,0,117,850,957,894,654,40,0,128,384,383,147,0,0,0,0,0,0,81,41,0,0,0],[0,0,0,175,891,940,802,503,106,27,242,362,313,44,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,223,922,871,617,357,180,181,283,325,233,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,252,931,629,300,241,214,237,283,274,141,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,252,884,0,0,144,207,241,256,211,39,0,0,0,0,0,0,0,0,0,0,0,34],[0,0,0,233,642,0,0,44,167,212,211,138,0,0,0,0,0,0,0,0,0,0,0,0,222],[0,0,0,705,12,0,0,0,102,161,152,57,0,0,0,0,0,0,0,0,0,0,0,0,363],[0,0,8,792,0,0,0,0,15,95,83,0,0,0,0,0,0,0,0,0,0,0,0,0,469],[0,0,0,0,0,0,0,0,0,19,10,0,0,0,0,0,0,0,0,0,0,0,0,112,551],[0,0,786,83,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,267,616],[0,0,961,392,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,39,383,669],[0,707,1000,555,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,174,473,713]],"best":[-3.1535,12.3287],"mean":[[1000,977,951,922,891,858,822,784,740,689,627,548,452,361,343,411,486,534,549,533,489,425,369,376,446],[991,967,939,908,875,840,803,763,718,663,595,506,390,273,271,379,474,528,545,527,477,395,307,294,385],[981,955,926,893,858,821,783,741,694,637,563,462,320,164,203,359,469,528,546,527,472,377,249,194,314],[971,943,912,877,840,801,760,717,669,610,531,419,248,46,167,358,474,535,552,533,476,374,212,82,240],[960,930,896,859,819,778,736,692,643,583,502,383,195,0,189,377,488,546,563,545,489,388,218,13,179],[948,916,880,839,797,753,710,666,617,559,479,363,193,94,256,411,509,563,579,561,509,416,263,65,163],[935,901,861,818,772,726,682,639,593,538,466,365,247,221,335,454,536,583,598,581,535,452,326,177,205],[922,884,841,794,744,696,652,611,571,525,465,391,324,330,411,500,567,607,619,605,563,492,391,282,276],[907,866,819,767,713,663,620,585,554,520,478,432,402,419,478,545,599,632,642,629,593,533,451,369,349],[891,847,795,737,679,626,587,562,543,525,502,480,471,492,538,589,632,659,667,655,623,572,506,441,417],[874,826,768,704,639,586,554,543,541,539,534,528,532,554,590,631,665,686,692,681,653,609,554,502,477],[855,802,738,666,595,543,524,531,547,560,568,574,586,607,637,669,696,713,717,707,682,644,598,554,529],[836,777,705,623,544,498,499,529,561,586,604,617,633,653,678,704,726,739,742,732,709,677,637,599,575],[815,750,668,572,485,453,484,536,581,615,639,657,674,694,715,737,754,765,765,756,736,707,672,639,616],[792,720,627,514,418,413,480,551,606,645,672,693,711,729,749,766,781,789,788,779,761,735,704,675,653],[769,688,580,444,343,388,490,573,632,674,703,725,744,762,779,794,805,811,810,801,784,761,734,707,686],[744,654,528,360,268,384,510,599,660,702,732,755,774,791,806,819,829,833,831,822,806,785,760,736,716],[717,617,469,257,214,402,538,627,687,729,759,782,801,817,831,842,850,853,851,842,827,808,785,762,744],[690,579,405,134,212,436,570,655,713,754,784,807,825,840,853,863,870,872,869,861,847,829,808,787,769],[662,539,335,21,259,477,602,683,738,778,808,830,848,862,874,883,889,890,887,878,865,848,829,809,792],[633,498,266,9,325,519,634,709,762,800,829,851,868,882,893,901,906,907,903,895,882,866,848,830,813],[605,458,210,105,391,560,664,734,784,821,849,870,886,900,910,918,922,922,918,910,898,883,866,849,833],[577,423,190,217,451,597,692,757,805,840,867,887,904,916,926,933,936,936,932,924,913,899,883,866,851],[551,396,217,313,503,631,717,779,824,858,883,904,919,931,941,947,950,950,946,938,927,914,898,883,868],[528,382,273,391,549,662,741,799,841,874,899,918,934,945,954,960,963,962,958,951,940,927,913,898,883]],"newest":[9.36362,2.24938],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0],[0.890363,6.32204],[-5.0,15.0],[3.23585,2.19754],[9.09776,2.7362],[9.79007,2.3748],[8.30079,0.0],[2.86339,2.49195],[-3.15455,12.1417],[-3.15626,12.5625],[9.49298,2.83877],[-3.1535,12.3287],[9.36362,2.24938]]}}, +{"best":0.00131003,"evaluations":26,"generation":20,"state":{"acquisition":[[0,0,0,79,324,467,523,490,320,0,0,0,569,873,702,287,270,55,0,17,0,0,0,0,0],[0,0,0,15,315,485,556,537,390,0,0,0,411,882,201,0,0,0,0,0,0,0,0,10,0],[0,0,0,0,317,514,600,597,485,128,0,0,41,903,0,177,0,0,0,0,38,0,182,301,0],[0,0,0,0,329,552,651,663,584,359,0,0,0,939,792,509,0,0,0,0,188,294,350,460,0],[0,0,0,0,352,597,705,727,671,528,353,401,395,960,862,509,0,0,0,0,246,326,0,939,0],[0,0,0,0,385,648,759,783,737,624,543,653,694,0,615,214,0,0,0,0,249,265,0,0,0],[0,0,0,0,428,700,809,830,777,648,551,654,628,0,0,0,0,0,0,0,226,203,0,58,0],[0,0,0,0,479,752,854,863,789,582,349,500,534,8,0,0,0,0,0,0,190,159,0,12,0],[0,0,0,0,535,802,890,883,764,337,0,279,489,167,0,0,0,0,0,0,142,112,0,0,0],[0,0,0,0,594,848,918,886,688,0,0,176,459,202,0,0,0,0,0,0,77,41,0,0,0],[0,0,0,0,656,888,935,868,527,0,0,196,425,185,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,716,920,938,819,253,0,0,237,380,137,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,774,942,920,729,0,0,0,259,325,65,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,828,948,869,586,0,0,0,257,258,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,875,925,757,404,0,0,73,234,180,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,39,910,838,534,229,0,0,129,193,90,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,60,922,533,164,93,43,68,135,138,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,48,869,0,0,0,42,82,110,69,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,31,598,0,0,0,3,56,63,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,686,0,0,0,0,0,3,0,0,0,0,0,0,0,0,0,0,0,0,0,0,154],[0,0,0,788,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,299],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,410],[0,0,757,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,45,497],[0,0,958,200,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,198,567],[0,673,1000,416,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,316,624]],"best":[-3.1535,12.3287],"mean":[[1000,975,947,917,885,851,815,777,734,684,622,545,451,364,345,409,484,532,548,534,492,424,358,361,443],[990,964,935,902,868,833,795,755,711,657,590,502,388,275,271,375,469,525,543,528,479,395,293,275,381],[980,952,921,887,850,813,774,732,686,630,557,458,318,166,202,354,464,524,544,527,474,378,235,169,313],[969,939,906,869,831,791,750,708,660,602,525,415,246,48,164,352,468,530,549,533,479,377,204,54,242],[957,925,890,850,809,767,725,682,634,575,496,379,193,2,185,371,482,541,561,545,492,393,220,0,183],[945,911,872,830,786,742,698,655,608,551,473,359,192,94,252,405,504,558,576,561,513,422,271,74,167],[931,895,853,808,760,713,669,628,584,531,461,362,245,219,331,449,531,579,595,582,538,459,336,188,204],[917,878,833,783,732,683,639,600,562,518,461,388,321,327,406,495,562,603,617,605,566,499,400,290,271],[902,860,810,756,700,649,607,574,546,514,474,428,397,414,473,540,595,629,641,630,596,538,458,373,342],[886,840,786,725,665,612,574,551,536,520,498,475,466,486,532,584,627,655,665,655,625,576,511,443,409],[869,818,759,692,625,571,542,533,535,534,529,523,526,547,583,624,659,682,690,680,654,612,557,501,468],[850,795,729,653,580,528,512,523,541,556,563,568,578,599,629,662,690,709,714,705,682,646,599,552,520],[831,770,696,610,529,483,489,522,556,581,598,610,624,644,670,697,720,734,738,730,709,677,637,596,566],[810,743,659,560,470,439,475,530,576,610,632,649,665,684,706,729,747,759,761,753,735,706,671,635,607],[788,714,618,502,404,402,474,546,601,639,665,684,702,720,739,758,773,783,784,776,759,733,702,670,644],[765,683,573,434,330,380,485,568,627,668,696,717,735,752,769,785,798,805,805,797,781,759,731,702,677],[740,649,522,351,258,378,506,594,654,695,725,746,764,781,796,810,820,826,825,818,803,782,757,730,707],[714,613,465,251,209,398,534,622,681,722,751,773,791,807,821,833,842,846,845,837,823,804,781,756,735],[688,575,402,132,210,432,565,650,707,747,776,798,815,830,843,854,862,865,863,855,842,824,803,781,760],[660,536,334,22,257,473,597,677,732,771,799,821,838,852,864,874,880,882,880,873,860,844,824,803,783],[631,496,265,11,323,515,628,703,755,793,821,842,858,872,883,892,897,899,896,889,877,861,843,823,805],[603,456,209,105,388,555,658,728,777,813,840,861,877,890,901,909,913,914,911,904,893,878,861,843,825],[574,420,186,215,447,592,686,751,798,833,859,879,895,907,917,924,928,929,926,919,908,894,878,860,844],[548,391,209,309,499,626,712,772,817,850,876,895,911,923,932,939,942,943,939,932,922,909,893,877,861],[524,374,264,386,544,657,736,793,835,867,892,911,926,937,946,952,955,955,952,945,935,923,908,892,877]],"newest":[-3.00111,11.9149],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0],[0.890363,6.32204],[-5.0,15.0],[3.23585,2.19754],[9.09776,2.7362],[9.79007,2.3748],[8.30079,0.0],[2.86339,2.49195],[-3.15455,12.1417],[-3.15626,12.5625],[9.49298,2.83877],[-3.1535,12.3287],[9.36362,2.24938],[-3.00111,11.9149]]}}, +{"best":0.00131003,"evaluations":27,"generation":21,"state":{"acquisition":[[0,0,0,0,224,389,454,418,225,0,0,0,498,852,658,176,158,0,0,0,0,0,0,0,0],[0,0,0,0,215,411,493,472,304,0,0,0,317,865,88,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,218,445,543,541,412,0,0,0,0,894,0,74,0,0,0,0,0,0,62,205,0],[0,0,0,0,233,489,602,615,524,262,0,0,0,942,775,448,0,0,0,0,84,201,260,395,0],[0,0,0,0,260,541,664,688,623,455,249,318,329,972,852,443,0,0,0,0,150,241,0,950,0],[0,0,0,0,299,599,725,753,699,564,467,606,661,0,571,100,0,0,0,0,153,173,0,0,0],[0,0,0,0,349,659,783,806,745,591,474,604,580,0,0,0,0,0,0,0,125,99,0,0,0],[0,0,0,0,407,719,834,845,759,514,230,425,468,0,0,0,0,0,0,0,81,42,0,0,0],[0,0,0,0,471,776,876,868,732,228,0,173,415,36,0,0,0,0,0,0,23,0,0,0,0],[0,0,0,0,539,828,908,872,646,0,0,58,380,78,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,608,874,927,851,464,0,0,79,339,60,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,676,910,930,795,142,0,0,123,288,6,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,742,935,909,685,0,0,0,144,223,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,802,941,848,497,0,0,0,138,146,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,854,916,707,227,0,0,0,109,56,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,890,815,400,0,0,0,0,59,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,892,454,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,771,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,25],[0,0,0,71,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,192],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,320],[0,0,733,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,419],[0,0,955,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,68,499],[0,622,1000,283,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,206,565]],"best":[-3.1535,12.3287],"mean":[[1000,974,946,915,882,848,812,774,731,681,620,543,450,364,345,408,482,530,546,532,490,422,356,359,440],[990,963,932,900,865,829,791,752,707,654,587,500,387,275,271,373,467,522,541,525,477,393,291,273,379],[979,950,918,883,846,808,769,728,682,626,554,455,317,166,201,352,461,521,541,525,472,376,233,167,311],[968,937,903,865,826,786,745,703,656,598,522,412,245,48,162,350,466,527,547,530,477,375,203,53,240],[956,923,886,846,804,762,720,677,629,571,493,376,192,2,184,369,480,539,558,542,490,391,219,0,183],[943,908,868,825,780,736,693,650,603,547,470,356,191,94,251,404,502,556,574,559,510,420,270,73,166],[929,892,849,802,754,707,663,622,579,527,457,359,244,218,330,447,529,577,593,579,536,457,334,187,203],[915,874,828,777,725,676,633,595,558,515,458,385,319,325,404,492,560,601,614,602,564,496,397,288,269],[900,856,805,750,693,642,600,569,541,511,471,426,395,411,470,538,592,626,638,627,593,535,456,371,340],[883,836,781,719,658,605,568,546,532,517,496,472,463,483,529,581,624,652,662,652,622,573,508,440,406],[866,814,754,685,618,564,536,529,531,532,526,520,522,543,580,621,656,679,686,677,651,609,554,498,465],[848,791,724,647,573,521,507,519,539,553,560,565,575,595,625,658,687,705,711,702,679,642,596,549,517],[828,766,691,604,521,476,484,519,554,579,595,607,620,640,666,693,716,731,734,726,705,674,634,593,563],[807,740,654,554,463,433,472,528,574,607,629,645,661,680,702,724,743,755,757,749,731,703,668,632,604],[785,711,614,496,397,397,471,544,599,636,662,681,697,715,735,753,769,778,780,772,755,730,699,667,641],[762,680,569,428,324,376,483,567,625,665,692,713,730,747,764,780,793,801,801,793,778,755,727,698,674],[738,646,518,346,252,376,504,592,652,693,721,742,760,776,791,805,816,822,821,814,799,778,753,727,705],[712,611,462,247,204,396,532,620,679,719,748,769,787,802,816,828,837,842,840,833,819,800,777,754,732],[686,573,400,130,206,431,563,647,704,744,773,794,811,826,838,849,857,860,859,851,839,821,800,778,758],[658,534,333,21,255,471,595,675,729,768,796,817,834,847,859,869,875,878,876,869,857,840,821,800,781],[630,494,265,11,320,512,626,701,752,790,817,838,854,868,879,887,893,895,892,885,874,858,840,821,803],[601,454,208,104,385,552,656,725,774,810,837,857,873,886,896,904,909,910,907,901,890,875,858,840,823],[573,417,184,213,444,589,683,748,795,830,855,875,891,903,913,920,924,925,922,915,905,891,875,858,842],[546,387,206,307,496,623,709,770,814,847,873,892,907,919,928,935,938,939,936,929,919,906,891,875,859],[521,370,259,383,542,654,733,790,832,864,889,907,922,933,942,948,951,952,948,942,932,920,906,891,876]],"newest":[3.13402,2.37865],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0],[0.890363,6.32204],[-5.0,15.0],[3.23585,2.19754],[9.09776,2.7362],[9.79007,2.3748],[8.30079,0.0],[2.86339,2.49195],[-3.15455,12.1417],[-3.15626,12.5625],[9.49298,2.83877],[-3.1535,12.3287],[9.36362,2.24938],[-3.00111,11.9149],[3.13402,2.37865]]}}, +{"best":0.00131003,"evaluations":28,"generation":22,"state":{"acquisition":[[0,0,0,0,86,282,356,311,78,0,0,0,397,817,564,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,80,310,404,377,175,0,0,0,191,840,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,87,352,465,458,304,0,0,0,0,880,0,0,0,0,0,0,0,0,0,67,0],[0,0,0,0,108,405,534,546,437,125,0,0,0,940,724,369,0,0,0,0,0,93,141,297,0],[0,0,0,0,143,467,607,631,553,353,109,133,0,885,821,353,0,0,0,0,48,150,0,962,0],[0,0,0,0,190,535,678,707,640,481,363,486,432,0,471,0,0,0,0,0,50,75,0,0,0],[0,0,0,0,249,605,745,768,694,512,366,467,321,0,0,0,0,0,0,0,14,0,0,0,0],[0,0,0,0,317,674,803,813,710,419,63,212,240,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,391,740,852,839,678,77,0,0,239,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,469,800,888,843,576,0,0,0,232,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,548,852,910,818,359,0,0,0,204,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,627,893,912,751,0,0,0,0,155,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,701,921,887,621,0,0,0,0,88,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,771,927,814,400,0,0,0,0,5,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,830,896,647,84,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,872,776,286,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,874,337,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,732,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,21],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,174],[0,0,700,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,293],[0,0,952,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,389],[0,578,1000,103,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,26,467]],"best":[-3.1535,12.3287],"mean":[[1000,973,944,912,879,844,808,770,728,678,617,541,449,363,345,408,481,528,544,530,488,420,354,356,438],[989,961,930,896,861,824,787,748,703,651,585,498,386,275,270,373,465,520,538,523,474,391,289,270,377],[978,948,915,879,842,803,764,724,678,623,551,453,315,166,199,350,459,519,538,522,469,374,232,166,309],[966,934,899,861,821,781,740,698,652,595,519,409,242,48,160,348,463,524,544,528,474,373,202,52,239],[954,920,882,841,799,756,714,672,625,568,489,373,189,3,182,367,477,536,555,539,487,389,217,0,182],[941,904,864,820,774,729,686,644,599,543,466,353,187,93,248,401,499,553,571,556,508,418,268,72,166],[927,888,844,796,748,700,657,617,575,524,454,356,240,216,327,444,526,574,590,576,533,454,332,185,202],[912,870,823,771,718,669,626,589,554,511,455,382,315,321,401,490,557,598,612,599,561,493,395,285,267],[897,852,800,743,686,634,594,564,538,508,468,422,390,407,467,534,589,623,635,624,590,533,453,368,338],[880,832,775,713,650,597,561,541,529,514,493,468,458,478,525,577,621,649,659,649,619,570,505,437,404],[863,810,748,678,610,556,529,525,528,529,523,515,517,538,575,617,653,676,683,674,648,606,551,495,462],[845,787,718,640,565,513,501,516,536,550,557,560,569,590,620,654,683,702,707,699,675,639,593,545,514],[825,762,685,597,513,469,480,516,551,576,591,602,615,634,660,688,711,727,731,723,702,670,630,589,560],[804,735,649,547,455,427,468,525,572,604,625,640,655,674,696,719,739,751,754,746,727,699,664,628,601],[783,707,609,490,389,392,468,542,596,633,657,675,692,709,729,748,764,774,776,768,751,726,695,663,638],[760,676,564,422,317,373,480,564,622,661,688,707,724,741,758,775,788,796,797,789,774,751,724,695,671],[735,643,515,341,247,374,502,590,648,688,716,737,754,770,785,800,811,817,817,810,795,775,750,724,702],[710,608,459,243,202,394,530,617,675,715,743,764,781,796,810,822,832,837,836,829,816,797,774,750,729],[684,571,398,128,205,428,560,644,700,740,768,788,805,820,833,844,852,855,854,847,835,817,796,775,755],[656,532,332,21,253,468,592,671,725,763,791,811,828,842,853,863,870,873,871,865,853,837,817,797,779],[628,492,264,11,318,510,623,697,748,785,812,832,848,862,873,882,888,890,888,881,870,855,837,818,801],[599,452,207,103,383,549,652,721,770,806,832,852,868,880,891,899,904,906,903,897,886,872,855,838,821],[571,415,182,212,442,586,680,744,791,825,851,870,885,898,907,915,919,920,918,911,901,888,872,856,840],[543,384,202,305,493,620,706,766,810,843,868,887,902,914,923,930,934,934,932,925,916,903,888,873,858],[519,365,255,381,539,651,730,786,828,860,884,903,917,929,937,944,947,948,945,939,930,918,904,889,874]],"newest":[3.20808,2.50564],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0],[0.890363,6.32204],[-5.0,15.0],[3.23585,2.19754],[9.09776,2.7362],[9.79007,2.3748],[8.30079,0.0],[2.86339,2.49195],[-3.15455,12.1417],[-3.15626,12.5625],[9.49298,2.83877],[-3.1535,12.3287],[9.36362,2.24938],[-3.00111,11.9149],[3.13402,2.37865],[3.20808,2.50564]]}}, +{"best":9.63498e-05,"evaluations":29,"generation":23,"state":{"acquisition":[[0,0,0,0,44,256,335,287,42,0,0,0,0,435,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,38,283,379,343,117,0,0,0,0,415,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,44,322,435,414,219,0,0,0,0,465,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,64,373,498,492,329,0,0,0,0,718,0,0,0,0,0,0,0,71,69,232,0],[0,0,0,0,96,432,567,570,432,54,0,0,0,0,0,0,0,0,0,0,46,144,0,970,0],[0,0,0,0,141,497,635,641,513,179,0,27,195,0,0,0,0,0,0,0,46,70,0,0,0],[0,0,0,0,198,566,701,702,564,194,0,205,296,0,0,0,0,0,0,0,2,0,0,0,0],[0,0,0,0,264,635,760,747,576,47,0,103,85,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,338,702,810,775,537,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,417,764,848,780,424,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,498,819,872,755,196,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,580,863,875,687,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,660,893,848,557,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,734,899,770,340,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,799,864,593,36,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,846,726,219,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,848,229,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,688,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,33],[0,0,686,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,167],[0,0,952,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,274],[0,558,1000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,362]],"best":[9.42451,2.48457],"mean":[[1000,972,942,910,876,841,806,768,727,677,615,533,430,329,323,403,479,525,540,526,485,419,353,355,438],[989,960,928,894,858,822,785,747,704,652,585,494,370,239,256,375,467,518,534,519,471,390,289,269,377],[978,947,913,877,839,801,763,724,680,626,555,454,305,132,201,361,464,518,535,518,466,373,232,165,309],[966,933,897,858,818,779,739,699,655,600,526,416,242,28,179,363,469,524,541,524,471,372,203,52,239],[954,919,880,838,796,754,714,674,630,576,500,384,197,6,206,382,483,536,553,536,484,387,217,0,182],[941,903,862,817,772,728,687,648,606,553,479,366,195,94,264,413,504,553,568,553,505,416,267,72,166],[927,887,842,794,745,699,658,621,583,534,466,366,238,207,333,451,530,574,588,573,530,452,330,184,202],[912,869,821,768,716,668,628,595,562,522,464,385,305,307,400,493,559,597,609,597,558,491,393,284,267],[897,851,798,741,684,634,597,570,546,516,473,418,375,391,461,534,589,622,633,621,588,531,451,367,337],[880,830,773,710,648,597,565,548,536,520,493,460,441,462,516,574,620,648,657,647,617,569,503,436,403],[863,809,746,676,608,557,534,531,534,531,519,504,500,522,565,612,650,674,681,672,646,604,550,494,461],[844,786,716,637,563,515,506,521,539,549,550,547,552,574,610,648,680,700,706,697,674,638,592,544,513],[825,761,683,594,512,471,484,520,552,572,582,588,598,619,650,682,708,725,729,721,701,669,629,588,559],[805,735,647,545,454,430,472,527,570,598,615,626,640,660,686,713,735,749,752,745,726,698,664,628,600],[783,706,607,488,388,395,471,541,592,625,647,662,677,696,719,742,760,772,774,767,751,726,695,663,637],[760,676,563,420,317,375,481,562,617,653,677,694,711,729,749,769,784,794,796,789,774,751,724,695,671],[736,643,514,340,248,375,501,586,643,680,706,725,741,759,777,794,807,815,816,809,795,775,750,724,702],[711,608,459,242,203,395,528,613,669,707,733,752,769,786,802,817,829,835,835,829,816,797,775,751,730],[685,571,398,127,206,428,558,640,694,732,758,778,795,811,825,839,849,854,854,848,835,818,797,775,755],[657,533,332,21,253,467,589,667,719,756,782,802,818,833,847,859,868,872,871,865,854,838,819,798,779],[629,493,264,11,318,508,620,693,743,778,804,824,840,855,867,878,886,889,888,882,871,856,838,819,802],[600,452,206,103,382,548,649,717,765,799,825,844,860,874,886,896,902,905,904,898,888,874,857,839,822],[571,415,180,212,441,585,677,741,786,819,844,863,879,892,903,912,918,920,919,913,903,890,874,858,842],[543,383,200,305,493,619,703,763,806,838,862,881,896,909,919,928,933,935,933,927,918,906,891,875,860],[518,363,252,381,539,650,728,784,825,856,879,898,913,925,935,942,947,948,946,941,932,920,906,891,877]],"newest":[9.42451,2.48457],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0],[0.890363,6.32204],[-5.0,15.0],[3.23585,2.19754],[9.09776,2.7362],[9.79007,2.3748],[8.30079,0.0],[2.86339,2.49195],[-3.15455,12.1417],[-3.15626,12.5625],[9.49298,2.83877],[-3.1535,12.3287],[9.36362,2.24938],[-3.00111,11.9149],[3.13402,2.37865],[3.20808,2.50564],[9.42451,2.48457]]}}, +{"best":9.63498e-05,"evaluations":30,"generation":24,"state":{"acquisition":[[0,0,0,0,0,147,251,202,0,0,0,0,0,241,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,184,305,271,17,0,0,0,0,228,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,233,371,352,131,0,0,0,0,312,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,292,444,439,251,0,0,0,0,670,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,361,521,525,362,0,0,0,0,0,0,0,0,0,0,0,0,39,0,910,0],[0,0,0,0,13,435,597,604,449,18,0,0,89,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,77,512,670,670,502,21,0,71,184,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,152,590,736,720,513,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,235,665,791,749,466,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,323,735,833,753,335,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,414,796,857,722,77,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,507,845,858,643,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,598,877,824,492,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,684,881,729,244,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,760,834,518,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,816,658,86,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,817,22,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,617,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,639,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,948,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,493,1000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,93]],"best":[9.42451,2.48457],"mean":[[1000,971,939,905,870,835,799,762,722,673,611,531,428,328,322,402,477,523,539,525,484,418,349,351,435],[989,958,924,888,852,815,778,740,698,647,582,492,368,239,255,374,465,516,533,518,470,389,285,265,375],[977,944,909,871,832,793,755,717,674,621,552,452,303,132,199,358,461,516,533,516,465,371,229,160,307],[965,930,892,852,811,770,731,692,649,596,523,414,240,27,178,361,466,521,539,522,470,370,200,50,238],[952,915,875,832,788,745,705,666,624,571,496,383,196,5,204,379,480,533,550,534,483,386,216,0,182],[939,900,856,810,763,718,678,640,600,549,476,364,194,93,263,411,501,550,566,551,503,415,266,70,165],[925,883,836,786,736,689,649,613,577,530,464,364,237,206,331,449,527,571,585,571,528,450,328,181,201],[910,865,815,760,706,658,619,587,557,518,462,383,302,305,397,490,556,595,607,594,556,489,391,281,265],[894,846,792,732,673,623,587,563,541,513,471,416,372,387,457,531,586,620,630,619,585,528,448,363,335],[878,826,766,701,637,586,555,541,532,517,490,456,436,457,512,571,617,645,654,644,615,566,500,432,400],[860,805,739,667,597,546,525,525,530,528,516,499,494,516,561,608,647,671,679,669,643,601,546,490,459],[842,782,710,629,552,504,498,516,536,546,546,542,546,568,604,644,676,696,703,694,671,635,588,541,511],[823,757,677,586,501,461,478,516,548,569,578,582,592,613,644,677,704,721,726,718,698,666,626,585,557],[802,731,642,536,443,421,466,523,567,594,610,620,633,653,680,707,730,745,749,742,723,695,660,624,598],[781,703,602,480,378,388,467,538,589,621,641,655,670,689,712,736,756,768,771,764,747,722,691,660,635],[758,673,559,413,308,370,478,559,613,648,671,688,703,722,742,763,780,790,792,785,770,748,720,692,669],[734,641,510,334,242,372,498,583,639,675,700,718,734,751,770,788,802,811,812,806,792,772,747,721,699],[709,606,456,238,199,392,525,609,664,701,727,745,762,778,795,811,823,831,832,825,813,794,771,748,728],[683,570,396,125,204,425,555,636,690,726,752,771,787,803,819,833,844,850,850,844,832,815,794,773,754],[656,531,331,20,252,465,586,662,714,750,776,795,811,826,840,853,863,868,868,862,851,835,816,796,778],[628,491,263,10,316,505,616,688,738,773,798,817,833,848,861,872,880,885,884,879,868,853,836,817,800],[599,451,205,102,380,544,645,713,760,794,819,838,853,867,880,890,897,901,900,895,885,871,855,837,821],[570,413,177,210,439,581,673,736,781,814,838,857,872,886,897,907,913,916,915,910,901,888,872,856,841],[542,380,195,303,491,616,700,759,801,833,857,875,890,903,914,922,928,931,930,925,916,904,889,874,859],[515,358,247,379,537,647,724,780,820,851,874,892,907,919,929,937,943,945,943,938,930,919,905,890,876]],"newest":[-3.83312,15.0],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0],[0.890363,6.32204],[-5.0,15.0],[3.23585,2.19754],[9.09776,2.7362],[9.79007,2.3748],[8.30079,0.0],[2.86339,2.49195],[-3.15455,12.1417],[-3.15626,12.5625],[9.49298,2.83877],[-3.1535,12.3287],[9.36362,2.24938],[-3.00111,11.9149],[3.13402,2.37865],[3.20808,2.50564],[9.42451,2.48457],[-3.83312,15.0]]}}, +{"best":2.13308e-05,"evaluations":31,"generation":25,"state":{"acquisition":[[0,0,0,0,0,147,251,202,0,0,0,0,0,241,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,184,305,271,17,0,0,0,0,228,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,233,371,352,131,0,0,0,0,312,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,292,444,439,251,0,0,0,0,670,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,361,521,525,362,0,0,0,0,0,0,0,0,0,0,0,0,39,0,910,0],[0,0,0,0,13,435,597,604,449,18,0,0,89,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,77,512,670,670,502,21,0,71,184,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,152,590,736,720,513,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,235,665,791,749,466,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,323,735,833,753,335,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,414,796,857,722,77,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,507,845,858,643,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,598,877,824,492,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,684,881,729,244,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,760,834,518,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,816,658,86,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,817,22,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,617,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,639,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,948,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,493,1000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,93]],"best":[9.42451,2.48457],"mean":[[1000,971,939,905,870,835,799,762,722,673,611,531,428,328,322,402,477,523,539,525,484,418,349,351,435],[989,958,924,888,852,815,778,740,698,647,582,492,368,239,255,374,465,516,533,518,470,389,285,265,375],[977,944,909,871,832,793,755,717,674,621,552,452,303,132,199,358,461,516,533,516,465,371,229,160,307],[965,930,892,852,811,770,731,692,649,596,523,414,240,27,178,361,466,521,539,522,470,370,200,50,238],[952,915,875,832,788,745,705,666,624,571,496,383,196,5,204,379,480,533,550,534,483,386,216,0,182],[939,900,856,810,763,718,678,640,600,549,476,364,194,93,263,411,501,550,566,551,503,415,266,70,165],[925,883,836,786,736,689,649,613,577,530,464,364,237,206,331,449,527,571,585,571,528,450,328,181,201],[910,865,815,760,706,658,619,587,557,518,462,383,302,305,397,490,556,595,607,594,556,489,391,281,265],[894,846,792,732,673,623,587,563,541,513,471,416,372,387,457,531,586,620,630,619,585,528,448,363,335],[878,826,766,701,637,586,555,541,532,517,490,456,436,457,512,571,617,645,654,644,615,566,500,432,400],[860,805,739,667,597,546,525,525,530,528,516,499,494,516,561,608,647,671,679,669,643,601,546,490,459],[842,782,710,629,552,504,498,516,536,546,546,542,546,568,604,644,676,696,703,694,671,635,588,541,511],[823,757,677,586,501,461,478,516,548,569,578,582,592,613,644,677,704,721,726,718,698,666,626,585,557],[802,731,642,536,443,421,466,523,567,594,610,620,633,653,680,707,730,745,749,742,723,695,660,624,598],[781,703,602,480,378,388,467,538,589,621,641,655,670,689,712,736,756,768,771,764,747,722,691,660,635],[758,673,559,413,308,370,478,559,613,648,671,688,703,722,742,763,780,790,792,785,770,748,720,692,669],[734,641,510,334,242,372,498,583,639,675,700,718,734,751,770,788,802,811,812,806,792,772,747,721,699],[709,606,456,238,199,392,525,609,664,701,727,745,762,778,795,811,823,831,832,825,813,794,771,748,728],[683,570,396,125,204,425,555,636,690,726,752,771,787,803,819,833,844,850,850,844,832,815,794,773,754],[656,531,331,20,252,465,586,662,714,750,776,795,811,826,840,853,863,868,868,862,851,835,816,796,778],[628,491,263,10,316,505,616,688,738,773,798,817,833,848,861,872,880,885,884,879,868,853,836,817,800],[599,451,205,102,380,544,645,713,760,794,819,838,853,867,880,890,897,901,900,895,885,871,855,837,821],[570,413,177,210,439,581,673,736,781,814,838,857,872,886,897,907,913,916,915,910,901,888,872,856,841],[542,380,195,303,491,616,700,759,801,833,857,875,890,903,914,922,928,931,930,925,916,904,889,874,859],[515,358,247,379,537,647,724,780,820,851,874,892,907,919,929,937,943,945,943,938,930,919,905,890,876]],"newest":[-3.83312,15.0],"polished":[9.42297,2.47587],"population":[[-2.09402,11.4832],[9.56653,8.17862],[1.77964,0.917823],[3.11662,3.15731],[-4.41227,6.40746],[5.12238,13.3845],[3.80246,1.0911],[5.75435,1.94742],[-4.08364,13.3527],[-2.25384,14.2432],[4.72756,5.29756],[10.0,3.80158],[10.0,0.0],[0.890363,6.32204],[-5.0,15.0],[3.23585,2.19754],[9.09776,2.7362],[9.79007,2.3748],[8.30079,0.0],[2.86339,2.49195],[-3.15455,12.1417],[-3.15626,12.5625],[9.49298,2.83877],[-3.1535,12.3287],[9.36362,2.24938],[-3.00111,11.9149],[3.13402,2.37865],[3.20808,2.50564],[9.42451,2.48457],[-3.83312,15.0]]}} +]} diff --git a/examples/bayesian_optimization/trace.py b/examples/bayesian_optimization/trace.py new file mode 100644 index 00000000..061fc4a2 --- /dev/null +++ b/examples/bayesian_optimization/trace.py @@ -0,0 +1,139 @@ +"""The trace of the run for the plot on the example's page, written to the file that +``GENOXIDE_TRACE`` names: a frame per step, with the points evaluated so far and, from the first +point the model chose, the model that chose it on a grid of 25 x 25 points: its posterior mean, +shaded on a log scale as the page shades the function, and the log expected improvement, the 25 +nats below its highest value shaded. Both are rounded to thousandths of their range, which is all +the page draws. A last frame shows the polished point. The Rust example writes the same file.""" + +import json +import math +import os + +import numpy as np + +import genoxide as gx + +# the points per side of the grid, and the span of the acquisition that is shaded, in nats +GRID = 25 +SPAN = 25.0 + + +class Trace: + """Records the run through ``control`` when ``GENOXIDE_TRACE`` is set.""" + + def __init__(self, minima, minimum): + self.path = os.environ.get("GENOXIDE_TRACE") + self.minima = minima + self.minimum = minimum + self.frames = [] + + def record(self, algorithm, progress): + """Records a step: the points so far, the newest one, the best, and the model that chose + the newest on the grid.""" + if not self.path: + return + points = progress.population.tolist() + state = { + "population": points, + "newest": points[-1], + "best": progress.best_genome.tolist(), + } + model = algorithm.model + if model is not None: + grid = points_of_grid() + mean = model.predict(grid)[0].reshape(GRID, GRID) + acquisition = algorithm.acquisition_at(grid).reshape(GRID, GRID) + state["mean"] = shaded(mean, lambda lo, hi, v: log1p(v - lo) / log1p(hi - lo)) + state["acquisition"] = shaded(acquisition, acquisition_shade) + self.frames.append( + { + "generation": progress.generation, + "evaluations": progress.evaluations, + "best": progress.best_fitness - self.minimum, + "state": state, + } + ) + + def write(self, x, value): + """Writes the trace, if there's one, with a last frame for the polished point ``x`` and + its value.""" + if not self.path: + return + last = json.loads(json.dumps(self.frames[-1])) + last["generation"] += 1 + last["evaluations"] += 1 + last["best"] = min(last["best"], value - self.minimum) + last["state"]["polished"] = [float(x[0]), float(x[1])] + self.frames.append(last) + settings = { + "format": 1, + "example": "bayesian_optimization", + "objective": "minimize", + "x_label": "evaluations", + "y_label": "error to the global minimum", + "log_y": True, + "optimum": 0.0, + "plot": "surrogate", + "problem": { + "function": "branin", + "bounds": [[-5.0, 10.0], [0.0, 15.0]], + "minima": self.minima.tolist(), + "grid": GRID, + }, + } + write(self.path, settings, self.frames) + + +def points_of_grid(): + """The grid over Branin's box, a point per row: x2 from 0 to 15 by row of the grid, x1 from + -5 to 10 by column, as the Rust example orders them.""" + + def at(low, high, i): + return low + (high - low) * i / (GRID - 1) + + return np.array( + [[at(-5.0, 10.0, column), at(0.0, 15.0, row)] for row in range(GRID) for column in range(GRID)] + ) + + +def log1p(x): + """genoxide's portable log1p, the Rust example's.""" + return float(gx.math.log1p(np.array([x]))[0]) + + +def acquisition_shade(lo, hi, v): + lo = max(lo, hi - SPAN) + return max((v - lo) / (hi - lo), 0.0) + + +def shaded(grid, shade): + """The grid's values as thousandths of their range: ``shade(lo, hi, v)`` in [0, 1], + rounded.""" + lo, hi = float(grid.min()), float(grid.max()) + return [[math.floor(1000.0 * shade(lo, hi, float(v)) + 0.5) for v in row] for row in grid] + + +# ---- the same in every example's trace ---------------------------------------------------------- + + +def write(path, settings, frames): + """Writes the settings and the frames to ``path``, a frame per line.""" + lines = ",\n".join(map(to_json, frames)) + with open(path, "w", encoding="utf-8", newline="\n") as file: + file.write(f'{to_json(settings)[:-1]},"frames":[\n{lines}\n]}}\n') + + +def to_json(value): + """Compact JSON with sorted keys, and numbers rounded to 6 significant digits, as the Rust + example writes it.""" + return json.dumps(rounded(value), sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def rounded(value): + if isinstance(value, dict): + return {key: rounded(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [rounded(item) for item in value] + if isinstance(value, float): + return float(f"{value:.5e}") if math.isfinite(value) else None + return value diff --git a/examples/bayesian_optimization/trace.rs b/examples/bayesian_optimization/trace.rs new file mode 100644 index 00000000..37bc7ec9 --- /dev/null +++ b/examples/bayesian_optimization/trace.rs @@ -0,0 +1,179 @@ +//! The trace of the run for the plot on the example's page, written to the file that +//! `GENOXIDE_TRACE` names: a frame per step, with the points evaluated so far and, from the first +//! point the model chose, the model that chose it on a grid of 25 × 25 points: its posterior mean, +//! shaded on a log scale as the page shades the function, and the log expected improvement, the +//! 25 nats below its highest value shaded. Both are rounded to thousandths of their range, which +//! is all the page draws. A last frame shows the polished point. The Python example writes the +//! same file. + +use genoxide::engine::Progress; +use genoxide::math::ln_1p; +use genoxide::prelude::*; +use genoxide::problems::{Branin, Problem}; +use serde_json::{Value, json}; + +// the points per side of the grid, and the span of the acquisition that is shaded, in nats +const GRID: usize = 25; +const SPAN: f64 = 25.0; + +pub struct Trace { + path: Option, + frames: Vec, + minimum: f64, +} + +impl Trace { + // a trace for the file that GENOXIDE_TRACE names, or nothing to record if it isn't set + pub fn from_env() -> Self { + let path = std::env::var("GENOXIDE_TRACE").ok(); + let minimum = Branin.optimum().expect("known").value(); + Self { + path, + frames: Vec::new(), + minimum, + } + } + + // records a step: the points so far, the newest one, the best, and the model that chose the + // newest on the grid + pub fn record(&mut self, bo: &Bo, progress: &Progress) { + if self.path.is_none() { + return; + } + let points: Vec<[f64; 2]> = bo + .population() + .iter() + .map(|individual| [individual.genome()[0], individual.genome()[1]]) + .collect(); + let best = bo.best().expect("evaluated").genome(); + let mut state = json!({ + "population": points, + "newest": points[points.len() - 1], + "best": [best[0], best[1]], + }); + if let Some(model) = bo.model() { + let mean = grid(|x| model.predict(x).mean()); + let acquisition = grid(|x| bo.acquisition_at(x).expect("a model")); + state["mean"] = json!(shaded(&mean, |lo, hi, v| ln_1p(v - lo) / ln_1p(hi - lo))); + state["acquisition"] = json!(shaded(&acquisition, |lo, hi, v| { + let lo = lo.max(hi - SPAN); + ((v - lo) / (hi - lo)).max(0.0) + })); + } + let error = progress.best().and_then(Fitness::score).expect("valid") - self.minimum; + self.frames.push(json!({ + "generation": progress.generation(), + "evaluations": progress.evaluations(), + "best": error, + "state": state, + })); + } + + // writes the trace, if there's one, with a last frame for the polished point `x` and its value + pub fn write(mut self, x: &[f64], value: f64) { + let Some(path) = self.path.take() else { return }; + let mut last = self.frames.last().expect("a frame").clone(); + let generation = last["generation"].as_u64().expect("a generation") + 1; + let evaluations = last["evaluations"].as_u64().expect("evaluations") + 1; + let best = last["best"] + .as_f64() + .expect("a best") + .min(value - self.minimum); + last["generation"] = json!(generation); + last["evaluations"] = json!(evaluations); + last["best"] = json!(best); + last["state"]["polished"] = json!([x[0], x[1]]); + self.frames.push(last); + let optimum = Branin.optimum().expect("known"); + let minima: Vec<&[f64]> = optimum.solutions().iter().map(|x| &x[..]).collect(); + let settings = json!({ + "format": 1, + "example": "bayesian_optimization", + "objective": "minimize", + "x_label": "evaluations", + "y_label": "error to the global minimum", + "log_y": true, + "optimum": 0.0, + "plot": "surrogate", + "problem": { + "function": "branin", + "bounds": [[-5.0, 10.0], [0.0, 15.0]], + "minima": minima, + "grid": GRID, + }, + }); + write(&path, settings, self.frames); + } +} + +// `f` on the grid over Branin's box: a row per value of x₂, from 0 to 15, and a column per value +// of x₁, from −5 to 10 +fn grid(f: impl Fn(&[f64]) -> f64) -> Vec> { + let at = |low: f64, high: f64, i: usize| low + (high - low) * i as f64 / (GRID - 1) as f64; + (0..GRID) + .map(|row| { + (0..GRID) + .map(|column| f(&[at(-5.0, 10.0, column), at(0.0, 15.0, row)])) + .collect() + }) + .collect() +} + +// the grid's values as thousandths of their range: `shade(lo, hi, v)` in [0, 1], rounded +fn shaded(grid: &[Vec], shade: impl Fn(f64, f64, f64) -> f64) -> Vec> { + let values = grid.iter().flatten(); + let lo = values.clone().fold(f64::INFINITY, |a, &b| a.min(b)); + let hi = values.fold(f64::NEG_INFINITY, |a, &b| a.max(b)); + grid.iter() + .map(|row| { + row.iter() + .map(|&v| (1000.0 * shade(lo, hi, v) + 0.5).floor() as u64) + .collect() + }) + .collect() +} + +// ---- the same in every example's trace --------------------------------------------------------- + +// writes the settings and the frames to `path`, a frame per line +fn write(path: &str, settings: Value, frames: Vec) { + let frames: Vec = frames.iter().map(to_json).collect(); + let settings = to_json(&settings); + let head = &settings[..settings.len() - 1]; + let text = format!("{head},\"frames\":[\n{}\n]}}\n", frames.join(",\n")); + std::fs::write(path, text).expect("the trace is written"); +} + +// compact JSON with sorted keys, and numbers rounded to 6 significant digits and written as +// Python writes them (7542.0, 1e-08): the Python example writes the same file +fn to_json(value: &Value) -> String { + let join = |items: Vec| items.join(","); + match value { + Value::Number(number) if number.is_f64() => python_float(number.as_f64().expect("f64")), + Value::Array(items) => format!("[{}]", join(items.iter().map(to_json).collect())), + Value::Object(map) => { + let entry = + |(key, item): (&String, &Value)| format!("{}:{}", json!(key), to_json(item)); + format!("{{{}}}", join(map.iter().map(entry).collect())) + } + other => other.to_string(), + } +} + +fn python_float(value: f64) -> String { + let rounded: f64 = format!("{value:.5e}").parse().expect("a number"); + let shortest = format!("{rounded:e}"); + let (mantissa, exponent) = shortest.split_once('e').expect("an exponent"); + let exponent: i32 = exponent.parse().expect("an exponent"); + if (-4..16).contains(&exponent) { + let text = rounded.to_string(); + if text.contains('.') { + text + } else { + text + ".0" + } + } else { + let sign = if exponent < 0 { '-' } else { '+' }; + format!("{mantissa}e{sign}{:02}", exponent.abs()) + } +} diff --git a/site/components/projects/genoxide/player/PlayerView.jsx b/site/components/projects/genoxide/player/PlayerView.jsx index 5758c668..12debab5 100644 --- a/site/components/projects/genoxide/player/PlayerView.jsx +++ b/site/components/projects/genoxide/player/PlayerView.jsx @@ -21,6 +21,7 @@ const PLOTS = { timeline: lazy(() => import("./plots/TimelinePlot")), grid: lazy(() => import("./plots/GridPlot")), cart_poles: lazy(() => import("./plots/CartPolesPlot")), + surrogate: lazy(() => import("./plots/SurrogatePlot")), }; const TITLES = { @@ -37,6 +38,7 @@ const TITLES = { surface: "The network's output", timeline: "Evaluations over time", cart_poles: "The best network at the controls", + surrogate: "The model and the point it chose", }; // the titles of a grid, by the kind of its panels diff --git a/site/components/projects/genoxide/player/plots/SurrogatePlot.jsx b/site/components/projects/genoxide/player/plots/SurrogatePlot.jsx new file mode 100644 index 00000000..5b02070a --- /dev/null +++ b/site/components/projects/genoxide/player/plots/SurrogatePlot.jsx @@ -0,0 +1,186 @@ +"use client"; + +import { useEffect, useId, useMemo, useState } from "react"; +import { categorical, formatValue, linear, resolveColor, ticks } from "../chart-kit"; +import { Axes, Legend, Marker, PlotBox, TipRows, Tooltip, nearest, pointerIn } from "../chart-parts"; +import { gridImage, isoline } from "../contour"; + +const LEVELS = [0.12, 0.24, 0.36, 0.48, 0.6, 0.72, 0.84]; + +/** + * `surrogate`: a step of Bayesian optimization in two panels. Left, the + * model's posterior mean over the box (`state.mean`, a grid of thousandths of + * its range on a log scale, row i along x₂, column j along x₁), darker where + * it's higher, with the points evaluated so far (`state.population`), the + * newest one (`state.newest`), the best (`state.best`), the known minima and, + * in the last frame, the point where the model's mean is lowest + * (`state.polished`). Right, the acquisition function that chose the newest + * point (`state.acquisition`, thousandths of its shaded range), darker where + * the point is more worth evaluating. Generation 0, the initial design, has no + * model: the left panel shows the points alone. + */ +export default function SurrogatePlot({ trace, frame, dark }) { + const problem = trace.problem ?? {}; + const bounds = problem.bounds ?? [ + [0, 1], + [0, 1], + ]; + const minima = problem.minima ?? []; + const state = frame.state ?? {}; + const palette = categorical(dark); + const legend = [ + { label: "evaluated", color: palette[1], shape: "dot" }, + { label: "newest", color: palette[2], shape: "square" }, + { label: "best", color: palette[0], shape: "diamond" }, + ...(state.polished ? [{ label: "the mean's minimum", color: palette[3], shape: "triangle" }] : []), + ...(minima.length ? [{ label: "global minima", shape: "ring", className: "text-base-content" }] : []), + ]; + return ( +
+ +
+ + +
+
+ ); +} + +function Panel({ title, shading, grid, bounds, minima, state, palette, dark, lines = false }) { + const [hover, setHover] = useState(null); + const [image, setImage] = useState(null); + const clip = `clip${useId().replace(/[^a-zA-Z0-9_-]/g, "")}`; + const n = grid?.length ?? 0; + const key = JSON.stringify(grid); + const values = useMemo(() => (key === "null" ? null : JSON.parse(key).map((row) => row.map((v) => v / 1000))), [key]); + const isolines = useMemo(() => (values && lines ? LEVELS.map((level) => isoline(values, level, (c) => c, (r) => r)) : []), [values, lines]); + + useEffect(() => { + if (!values) { + setImage(null); + return; + } + const [r, g, b] = resolveColor("var(--color-base-content)"); + const top = dark ? 0.42 : 0.36; + setImage(gridImage(values, (t) => [r, g, b, Math.round(255 * (0.02 + top * t))])); + }, [values, dark]); + + const population = state.population ?? []; + const inBounds = (p) => p && p[0] >= bounds[0][0] && p[0] <= bounds[0][1] && p[1] >= bounds[1][0] && p[1] <= bounds[1][1]; + + return ( +
+

+ {title} + {values ? · {shading} : null} +

+ + hover ? ( + + + + ) : null + } + > + {({ width, height }) => { + const margin = { left: 40, right: 12, top: 12, bottom: 38 }; + const side = Math.min(width - margin.left - margin.right, height - margin.top - margin.bottom); + const left = margin.left + (width - margin.left - margin.right - side) / 2; + const area = { left, right: left + side, top: margin.top, bottom: margin.top + side }; + const x = linear(bounds[0], [area.left, area.right]); + const y = linear(bounds[1], [area.bottom, area.top]); + const cell = n > 1 ? side / (n - 1) : side; + const points = population.filter(inBounds).map((p) => ({ p, px: x(p[0]), py: y(p[1]) })); + return ( + <> + + + + + + + {image ? ( + + ) : null} + {isolines.length ? ( + + + {isolines.map((d, k) => ( + + ))} + + + ) : null} + + + {minima.map((m) => ( + + ))} + {points.map((q, k) => ( + + ))} + {inBounds(state.best) ? : null} + {inBounds(state.newest) ? : null} + {inBounds(state.polished) ? : null} + + {hover ? : null} + { + const p = pointerIn(event); + const hit = nearest(points, p.x, p.y, 14); + setHover(hit ? { p: hit.p, px: hit.px, py: hit.py } : null); + }} + onPointerLeave={() => setHover(null)} + /> + + ); + }} + +
+ ); +} From 85470cc26aaab57c44bcaaefe44b3cf1f2a4081a Mon Sep 17 00:00:00 2001 From: tachsin Date: Fri, 2 Oct 2026 12:39:56 +0300 Subject: [PATCH 09/13] feat(python): Bo, its running handle and gx.model.gp gx.Bo(real, ...) with the Rust builder's settings, the acquisitions as "log-ei", "ei", gx.ProbabilityOfImprovement(xi) and gx.UpperConfidenceBound(beta), and gx.RunningBo for a control: the acquisition, which can change, the model that chose the last point and the acquisition's values. gx.model.gp.GaussianProcess fits and queries the Rust model on numpy arrays. A run in Python gives the bits of the same run in Rust, which tests on both sides check. --- python/README.md | 27 +++ python/genoxide/__init__.py | 253 ++++++++++++++++++++++++++- python/genoxide/_genoxide.pyi | 22 +++ python/genoxide/model/__init__.py | 9 + python/genoxide/model/gp.py | 235 +++++++++++++++++++++++++ python/src/config.rs | 51 ++++++ python/src/control.rs | 111 ++++++++++++ python/src/lib.rs | 2 + python/src/model.rs | 234 +++++++++++++++++++++++++ python/src/run.rs | 61 ++++++- python/tests/test_bo.py | 273 ++++++++++++++++++++++++++++++ python/tests/test_docs.py | 2 +- tests/bo.rs | 14 ++ 13 files changed, 1285 insertions(+), 9 deletions(-) create mode 100644 python/genoxide/model/__init__.py create mode 100644 python/genoxide/model/gp.py create mode 100644 python/src/model.rs create mode 100644 python/tests/test_bo.py diff --git a/python/README.md b/python/README.md index 79462d10..90c6856b 100644 --- a/python/README.md +++ b/python/README.md @@ -114,6 +114,7 @@ An exception in the fitness function stops the run, and `run` raises it. So does | Yes / no choices (subsets) | `Binary` | `Ga` with `UniformCrossover()` or `PointCrossover(points)`, and `BitFlip` | | An order (tours, sequencing) | `Permutation` | `LocalSearch`, which often beats a GA on permutations; `Ga` with `OrderCrossover()` (sequences) or `EdgeRecombinationCrossover()` (tours) | | Reals in ranges | `Real` | `Cmaes`; `De`; `Es`; `Ga` with `SimulatedBinaryCrossover(eta)` and `PolynomialMutation(eta)`; `Lbfgsb` for a local minimum of a smooth function, any number of genes; `NelderMead` for a local minimum in a few dimensions | +| An expensive function of a few reals (tens to a few hundred evaluations) | `Real` | `Bo`, Bayesian optimization, up to about 10 to 20 genes | | A smooth function of many reals, with its gradient | `Real` | `FirstOrder` (Adam, momentum, Nesterov), up to millions of genes | | Smooth, with gradients: very many reals (up to millions), few inequality constraints | `Real` | `Mma`; `method="gcmma"` to converge from any start | | A neural network's weights | `Real`, from `network.representation(bounds)` | `Cmaes` up to a few hundred weights; `OpenEs` for thousands and more | @@ -131,6 +132,7 @@ An exception in the fitness function stops the run, and `run` raises it. So does - `FirstOrder` steps along the gradient of a smooth function by a step rule, without a line search: `step="adam"` (Kingma and Ba's, the default), `"adamw"` (with decoupled weight decay), `"momentum"`, `"nesterov"` or `"gradient"`. One gradient per generation, from `run(..., gradient=...)` as for `Lbfgsb`, from a problem of `genoxide.problems` in Rust, or by finite differences (`n` more evaluations per generation, up to 10,000 genes). Memory and work per step are linear in the genes, for up to millions of them. It stops with the stop reason `"converged"` when the gradient vanishes; a `control` lowers the learning rate over the run (a schedule), which Adam needs to settle on the minimum. - `Mma`, Svanberg's method of moving asymptotes, uses the gradient of the score and of each inequality constraint `g(x) <= 0`: the gradient as for `Lbfgsb` (no finite differences), and with `run(f, gradient=True, constraints=m)`, `f` returns `(value, gradient, g, jacobian)`, `g` the constraints' values and `jacobian` an array of a row per constraint. An iteration is one evaluation and a few passes over the genes, with no matrix of them: for up to millions of genes and up to a few hundred constraints. It stops on its own once it has converged (the stop reason `"converged"`). `method="gcmma"` converges from any start, at the cost of more evaluations; `constraint_cost` must exceed the constraints' multipliers. - `Continuation` runs `FirstOrder`, `Lbfgsb` or `Mma` through stages of one problem, a smooth version first and sharper ones after (a smoothing that shrinks, a p-norm's p that grows, a penalty raised): `on_stage(index)` sets the stage's parameters for the fitness function, and the method goes on from its point with its state (Adam's averages, MMA's asymptotes; `keep="point"` keeps only the point). A stage ends when the method converges or after `generations`; the run, after the last stage. +- `Bo`, Bayesian optimization, is for expensive functions, where each evaluation counts: a Gaussian process (`gx.model.gp`) models the function from every evaluation, and an acquisition function of its posterior chooses the next point, one per generation after an initial design of `2(n + 1)` points. The log expected improvement by default; `"ei"`, `gx.ProbabilityOfImprovement(xi)` and `gx.UpperConfidenceBound(beta)` (whose `beta` a `control` can change). `output="log"` models the logarithm of the values, for objectives that span orders of magnitude. The model's fit costs O(N^3) for N evaluations: up to a few hundred evaluations, in up to about 10 to 20 genes. A running `Bo` gives its `model` and `acquisition_at(points)` to a `control`, e.g. for a plot. - `Islands` of `Ga`s or `De`s evolve apart and exchange their best: more diverse than one large population, and often faster on multimodal problems. ## Algorithms @@ -146,6 +148,7 @@ An exception in the fitness function stops the run, and `run` raises it. So does | `Neat` | its networks | `inputs`, `outputs` (needed), `population_size` (150), `compatibility` (`(1.0, 1.0, 0.4, 3.0)`: c1, c2, c3, threshold), `weight_mutation` (`(0.8, 0.1)`: rate, replace), `weight_deviations` (`(1.0, 1.0)`), `structural_mutation` (`(0.03, 0.05)`: add node, add connection), `reproduction` (`(0.25, 0.001, 0.75)`), `selection` (`(5, 0.2)`: elitism size, survival), `stagnation` (15), `activation` (`"steep_sigmoid"`), `feed_forward` (True), `initial` (`"fully_connected"`; `"unconnected"`), `sharing` (`"normalized"`; `"raw"`, the paper's) | | `Pso` | real | `population_size` (needed), `ring` (neighbors on each side) | | `NelderMead` | real | `coefficients` (`"adaptive"`, Gao and Han's; `"standard"`; `(reflection, expansion, contraction, shrink)`), `initial_step` (0.1 of each range) or `initial_step_absolute` (a distance), `tolerance` (1e-9 of the initial step), `restarts` (none; random restarts), `speculative` (False), `initial_genome` (a random point) | +| `Bo` | real | `initial_points` (2(n + 1)), `initial_genomes` (none), `acquisition` (`"log-ei"`; `"ei"`, `ProbabilityOfImprovement(xi)`, `UpperConfidenceBound(beta)`), `kernel` (`"matern52"`; `"squared_exponential"`), `noise` (0: the model interpolates; a fixed fraction of the values' variance, or `gx.model.gp.Learned(min)`), `output` (`"standardize"`; `"log"`), `raw_samples` (1000), `acquisition_starts` (10), `hyperparameter_starts` (5) | | `Lbfgsb` | real | `memory` (10), `gradients` (`"auto"`; `"supplied"`, `"forward"`, `"central"`), `difference_step` (√ε forward, ε^(1/3) central), `gradient_tolerance` (1e-5), `function_tolerance` (2.2e-9), `max_line_search` (20), `restarts` (none; random restarts), `initial_genome` (a random point) | | `Continuation` | real | `algorithm` (a `FirstOrder`, `Lbfgsb` or `Mma`), `stages`, `on_stage` (needed), `generations` (none: each stage to convergence), `keep` (`"state"`; `"point"`), `on_stage_finished` (none); `Lbfgsb(keep_pairs=True)` keeps its pairs between stages; the result's `stages` | | `Mma` | real | `method` (`"mma"`; `"gcmma"`), `asymptote_initial` (0.5 of each range), `asymptote_decrease` (0.7), `asymptote_increase` (1.2), `move_limit` (0.5 of each range), `constraint_cost` (1000), `kkt_tolerance` (1e-9), `step_tolerance` (1e-10 of each range), `restoration` (True), `parallel_sums` (False), `initial_genome` (a random point); `run(f, gradient=True, constraints=m, ...)` | @@ -586,6 +589,27 @@ print(result.stop_reason, [s.generations for s in result.stages]) # converged [ assert np.allclose(result.best_genome, target, atol=1e-6) ``` +`Bo` spends tens of evaluations where the others spend thousands. Its model is a Gaussian process of `gx.model.gp`, which can also be fitted on its own: here to the run's evaluations, its mean minimized by `Lbfgsb` with its gradient to polish the best point, for one evaluation more: + +```python +problem = gx.problems.Branin() +evaluated = {} + +def keep(progress): + evaluated["points"], evaluated["values"] = progress.population, progress.scores + +bo = gx.Bo(problem.genome, objective="minimize", seed=1) +result = bo.run(problem, evaluations=30, on_generation=keep) +model = gx.model.gp.GaussianProcess.fit(problem.genome, evaluated["points"], evaluated["values"]) +polish = gx.Lbfgsb(problem.genome, initial_genome=result.best_genome, objective="minimize") +polished = polish.run( + lambda x: model.predict(x)[0][0], + gradient=lambda x: model.predict_with_gradient(x)[2], + evaluations=1_000, +) +print(result.best_fitness, problem(polished.best_genome)) # 0.397984 0.397909, the minimum 0.397887 +``` + ## Progress `run(..., on_generation=callback)` calls `callback` after every generation, the initial population (generation 0) included. It runs on the thread that called `run`. It gets a read-only object: @@ -615,6 +639,7 @@ result = ga.run(lambda bits: bits.sum(), generations=1_000, on_generation=report | `Pso` | `RunningPso` | `inertia`, `acceleration` (`(cognitive, social)`) | | `LocalSearch` | `RunningLocalSearch` | `neighbor`, `neighbors` | | `NelderMead` | `RunningNelderMead` | none: its steps follow from its simplex. It reads `converged`, `size` (of the simplex, a fraction of each range), `iterations` and `restart_count` | +| `Bo` | `RunningBo` | `acquisition` (e.g. `UpperConfidenceBound(beta)` with `beta` on a schedule). It reads `initial_points`, `model` (the `gx.model.gp.GaussianProcess` that chose the last point, None in generation 0) and `acquisition_at(points)` | | `Lbfgsb` | `RunningLbfgsb` | `memory`. It reads `pairs`, `converged` (the criterion), `projected_gradient`, `iterations`, `gradients` (the source in use), `gradient_evaluations` and `stencil_evaluations` (the cost of finite differences), `skipped_pairs`, `memory_resets` and `restart_count` | | `Mma` | `RunningMma` | none: its steps follow from its approximations. It reads `converged` (`"kkt"`, `"step"` or None), `iterations`, `inner_iterations`, `multipliers` and `kkt_residual` | | `Continuation` | the wrapped method's | as the wrapped method | @@ -768,6 +793,8 @@ Some names differ: | `Lbfgsb(gradients="central", difference_step=h)` | `.gradients(Gradients::Central { step: Some(h) })` | | `Lbfgsb.run(f, gradient=g)`, `gradient=True` | `Differentiable(\|x, gradient\| ...)` | | `Mma(method="gcmma")` | `.method(mma::Method::Gcmma)` | +| `Bo(acquisition=gx.UpperConfidenceBound(beta))`, `Bo(output="log")`, `Bo(noise=gx.model.gp.Learned(1e-6))` | `.acquisition(bo::Acquisition::UpperConfidenceBound { beta })`, `.output(bo::Output::Log)`, `.noise(model::gp::Noise::Learned { min: 1e-6 })` | +| `gx.model.gp.GaussianProcess.fit(genome, points, values)`, `model.predict(points)` | `GaussianProcess::builder(real).fit(&points, &values)?`, `model.predict(&x)` | | `Mma.run(f, gradient=True, constraints=m)`, `f` returning `(value, gradient, g, jacobian)` | `Constrained::differentiable(m, \|x, gradient, g, jacobian\| value)` | | `Continuation(method, stages=n, on_stage=f, generations=g, keep="point")`, `result.stages` | `Continuation::builder(method).stages(n).on_stage(\|stage, _\| ...).generations(g).keep(Keep::Point)`, `continuation.stages()` | | `FirstOrder(step="adam", learning_rate=a)`, `FirstOrder(step="nesterov", learning_rate=a, momentum=m)`, `FirstOrder(step="adamw", learning_rate=a, weight_decay=w)` | `.step(first_order::Step::adam(a))`, `.step(first_order::Step::nesterov(a, m))`, `.step(first_order::Step::adamw(a, w))` | diff --git a/python/genoxide/__init__.py b/python/genoxide/__init__.py index 5c13f664..62d8e347 100644 --- a/python/genoxide/__init__.py +++ b/python/genoxide/__init__.py @@ -1,7 +1,8 @@ """Evolutionary computation in Rust, for Python. Genetic algorithms, local search, the Nelder-Mead simplex method, L-BFGS-B and first-order -gradient methods (gradient descent, momentum, Nesterov, Adam and AdamW), differential evolution, +gradient methods (gradient descent, momentum, Nesterov, Adam and AdamW), Bayesian optimization +(with the Gaussian processes of :mod:`genoxide.model.gp`), differential evolution, evolution strategies, CMA-ES, OpenAI's evolution strategy, NEAT, particle swarm optimization, the island model, and NSGA-II, NSGA-III, SPEA2, MOEA/D and SMS-EMOA for several objectives, from `genoxide `_, with Python fitness functions:: @@ -134,6 +135,9 @@ "Pso", "LocalSearch", "NelderMead", + "Bo", + "ProbabilityOfImprovement", + "UpperConfidenceBound", "FirstOrder", "Lbfgsb", "Mma", @@ -164,6 +168,7 @@ "RunningPso", "RunningLocalSearch", "RunningNelderMead", + "RunningBo", "RunningFirstOrder", "RunningLbfgsb", "RunningMma", @@ -175,6 +180,7 @@ "neat", "math", "gp", + "model", ] ObjectiveName = Literal["maximize", "minimize"] @@ -1357,9 +1363,10 @@ class Running: Each algorithm has a class of its own, with its settings as properties: :class:`RunningGa`, :class:`RunningDe`, :class:`RunningEs`, :class:`RunningCmaes`, :class:`RunningOpenEs`, :class:`RunningNeat`, :class:`RunningPso`, :class:`RunningLocalSearch`, - :class:`RunningNelderMead`, :class:`RunningLbfgsb`, :class:`RunningFirstOrder`, - :class:`RunningMma` and :class:`RunningIslands`. A new value is checked as in the algorithm's constructor: a wrong one raises a ``ValueError`` and changes nothing. The handle - works only during the callback; afterwards it raises a ``RuntimeError``. + :class:`RunningNelderMead`, :class:`RunningBo`, :class:`RunningLbfgsb`, + :class:`RunningFirstOrder`, :class:`RunningMma` and :class:`RunningIslands`. A new value is + checked as in the algorithm's constructor: a wrong one raises a ``ValueError`` and changes + nothing. The handle works only during the callback; afterwards it raises a ``RuntimeError``. A control that changes nothing leaves the run as it is: with a seed, the same result as without the control. @@ -1671,6 +1678,48 @@ def restart_count(self) -> int: return int(self._get("restart_count")) +class RunningBo(Running): + """A running :class:`Bo`, for ``control``: its acquisition function, which can change (e.g. + UCB's ``beta`` on a schedule), and the model that chose the last point, with the + acquisition's values, e.g. for a plot. ``reevaluate()`` asks every evaluated point again.""" + + __slots__ = () + + @property + def acquisition(self) -> BoAcquisition: + """The acquisition function: "log-ei", "ei", a :class:`ProbabilityOfImprovement` or an + :class:`UpperConfidenceBound`. A new one applies from the next point.""" + return _acquisition_of(self._get("acquisition")) + + @acquisition.setter + def acquisition(self, value: BoAcquisition) -> None: + self._set("acquisition", _acquisition(value)) + + @property + def initial_points(self) -> int: + """The points of the initial design.""" + return int(self._get("initial_points")) + + @property + def model(self) -> _surrogate.GaussianProcess | None: + """The Gaussian process that chose the last point, fitted to the evaluations before it: + a model of the values the search minimizes (the scores, negated when maximizing, through + ``output``). None in generation 0, and after resuming from a checkpoint until the next + point.""" + native = self._native.model() + return None if native is None else _surrogate.GaussianProcess(native) + + def acquisition_at(self, points: Any) -> np.ndarray: + """The acquisition function under :attr:`model` at ``points``, a point per row (a 1-D + array is one point), as the search maximizes it: the log expected improvement, the + expected improvement, the logarithm of the probability of improvement or the negated + lower confidence bound, in the model's standardized units. Raises a ``ValueError`` + without a model.""" + rows = np.ascontiguousarray(points, dtype=np.float64) + if rows.ndim == 1: + rows = rows.reshape(1, -1) + return np.asarray(self._native.acquisition(rows)) + class RunningLbfgsb(Running): """A running :class:`Lbfgsb`, for ``control``: its ``memory``, which can change, and its state, read-only. ``reevaluate()`` scores the current point again and drops the correction @@ -2308,8 +2357,8 @@ def run( ``on_generation``, on the same thread, with the running algorithm (a :class:`RunningGa`, :class:`RunningDe`, :class:`RunningEs`, :class:`RunningCmaes`, :class:`RunningOpenEs`, :class:`RunningPso`, :class:`RunningLocalSearch`, - :class:`RunningNelderMead`, :class:`RunningLbfgsb`, :class:`RunningFirstOrder`, - :class:`RunningMma` or :class:`RunningIslands`) and a + :class:`RunningNelderMead`, :class:`RunningBo`, :class:`RunningLbfgsb`, + :class:`RunningFirstOrder`, :class:`RunningMma` or :class:`RunningIslands`) and a :class:`Progress`: to change the algorithm's settings for the next generation, or to re-evaluate it after the fitness function changed. See :class:`Running`. @@ -3548,6 +3597,196 @@ def _describe(self) -> dict[str, Any]: } +@dataclass(frozen=True) +class ProbabilityOfImprovement: + """Bayesian optimization's probability of improving on the best value by more than ``xi`` + (Kushner 1964), maximized through its logarithm, which has the same maximizer. ``xi`` is 0 or + more, in the units the model fits (the scores, or their logarithm with ``output="log"``): + larger explores more.""" + + xi: float + + def _describe(self) -> dict[str, Any]: + return {"type": "probability_of_improvement", "xi": _number("xi", self.xi)} + + +@dataclass(frozen=True) +class UpperConfidenceBound: + """Bayesian optimization's confidence bound ``mean - sqrt(beta) sd``, minimized (``mean + + sqrt(beta) sd`` maximized when maximizing): Srinivas, Krause, Kakade and Seeger (2010). + ``beta`` is 0 or more: 0 exploits the model's mean alone, larger explores more.""" + + beta: float + + def _describe(self) -> dict[str, Any]: + return {"type": "upper_confidence_bound", "beta": _number("beta", self.beta)} + + +BoAcquisition = Union[Literal["log-ei", "ei"], ProbabilityOfImprovement, UpperConfidenceBound] + + +def _acquisition(acquisition: Any) -> dict[str, Any]: + if isinstance(acquisition, (ProbabilityOfImprovement, UpperConfidenceBound)): + return acquisition._describe() + if isinstance(acquisition, str) and acquisition == "log-ei": + return {"type": "log_expected_improvement"} + if isinstance(acquisition, str) and acquisition == "ei": + return {"type": "expected_improvement"} + raise ValueError( + 'acquisition is "log-ei", "ei", ProbabilityOfImprovement(xi) or ' + f"UpperConfidenceBound(beta), not {acquisition!r}" + ) + + +def _acquisition_of(description: dict[str, Any]) -> BoAcquisition: + kind = description["type"] + if kind == "probability_of_improvement": + return ProbabilityOfImprovement(float(description["xi"])) + if kind == "upper_confidence_bound": + return UpperConfidenceBound(float(description["beta"])) + return "ei" if kind == "expected_improvement" else "log-ei" + + +class Bo(_SingleObjective): + """Bayesian optimization. Real genomes. + + For expensive black-box functions, such as a simulation that runs for minutes or a physical + experiment, where tens to a few hundred evaluations must do. A Gaussian process + (:mod:`genoxide.model.gp`) models the function from every evaluation so far, and an + acquisition function of its posterior picks the next point, trading the model's best guesses + against its uncertainty. Generation 0 evaluates an initial design: the ``initial_genomes``, + and a Latin hypercube (McKay, Beckman and Conover 1979) for the rest of ``initial_points``. + Each later generation fits the model and evaluates one point, the acquisition's maximum: from + ``raw_samples`` random points, then L-BFGS-B with the acquisition's gradient from the best + ``acquisition_starts`` of them and from the best point so far. A point is never evaluated + twice. The model fits the scores to minimize (negated when maximizing), through ``output``; + an invalid score enters it at the worst value of the others, so the search learns to avoid + where the function fails, and a constraint violation is ignored by the search (use a + penalty). The model's fit costs O(N^3) for N evaluations: up to a few hundred evaluations of + a function far more expensive than that, in up to about 10 to 20 genes. + + Branin's function, whose three global minima are 0.397887, in 40 evaluations:: + + import genoxide as gx + + problem = gx.problems.Branin() + bo = gx.Bo(problem.genome, objective="minimize", seed=1) + result = bo.run(problem, evaluations=40) + print(result.best_fitness) # 0.3978873... + + The model, its likelihood's maximization and the acquisition's are seeded and portable: a + seed repeats the run on every platform, and gives a Rust program's results. + + Parameters + ---------- + genome : Real + The search space. At least one gene needs ``low < high``. + initial_points : int, optional + The points of the initial design, at least 1. None is 2(n + 1) for the n genes with + ``low < high``: a small design leaves most evaluations to the model. 10n is the usual + size for an accurate model of the whole box (Loeppky, Sacks and Welch 2009), more than + finding a minimum needs. + initial_genomes : 2-D array-like of float, optional + Genomes evaluated first, in the initial design, a genome per row: at most + ``initial_points`` of them, distinct, within the bounds. + acquisition : "log-ei", "ei", ProbabilityOfImprovement or UpperConfidenceBound + The acquisition function, "log-ei" by default: the logarithm of the expected + improvement, computed so that it and its gradient stay finite where the expected + improvement underflows (Ament et al. 2023), the same maximizer found far more reliably + than with "ei", the expected improvement itself (Mockus 1975; Jones, Schonlau and Welch + 1998). + kernel : "matern52" or "squared_exponential", default "matern52" + The model's kernel, with a length scale per gene. + noise : float or genoxide.model.gp.Learned, default 0.0 + The model's noise variance, a fraction of the values' variance: 0 interpolates the + values, as suits a deterministic function; :class:`genoxide.model.gp.Learned` learns it, + for a noisy one. + output : "standardize" or "log", default "standardize" + What the model fits: the values themselves, standardized; or ``log(v - v_best + d)`` of + the values to minimize, ``d`` the first quartile of their distances above the best, for + objectives that span orders of magnitude (Goldstein-Price's 3 to 10^6). + raw_samples : int, default 1000 + The random points at which the acquisition is evaluated before its maximization, at + least 1. + acquisition_starts : int, default 10 + The best raw samples from which L-BFGS-B maximizes the acquisition, at least 1 and at + most ``raw_samples``. + hyperparameter_starts : int, default 5 + The starts of the likelihood's maximization at each fit, at least 1: the last fit's + hyperparameters, then random ones. + objective : {"maximize", "minimize"}, default "maximize" + Whether higher or lower scores are better. + seed : int, optional + The seed of the random numbers (the design, the starts of both maximizations), 0 to + 2^64 - 1. None is a random seed. The same seed repeats the run. + + References: Jones, D. R., Schonlau, M. and Welch, W. J. (1998). Efficient global optimization + of expensive black-box functions. *Journal of Global Optimization* 13(4): 455-492. Rasmussen, + C. E. and Williams, C. K. I. (2006). *Gaussian Processes for Machine Learning.* MIT Press. + Ament, S., Daulton, S., Eriksson, D., Balandat, M. and Bakshy, E. (2023). Unexpected + improvements to expected improvement for Bayesian optimization. *NeurIPS 2023*. + """ + + _running = RunningBo + + def __init__( + self, + genome: Real, + *, + initial_points: int | None = None, + initial_genomes: Sequence[Sequence[float]] | np.ndarray | None = None, + acquisition: BoAcquisition = "log-ei", + kernel: Literal["matern52", "squared_exponential"] = "matern52", + noise: Any = 0.0, + output: Literal["standardize", "log"] = "standardize", + raw_samples: int = 1000, + acquisition_starts: int = 10, + hyperparameter_starts: int = 5, + objective: ObjectiveName = "maximize", + seed: int | None = None, + ) -> None: + self._genome = genome + self._objective = objective + self.initial_points = initial_points + self.initial_genomes = initial_genomes + self.acquisition = acquisition + self.kernel = kernel + self.noise = noise + self.output = output + self.raw_samples = raw_samples + self.acquisition_starts = acquisition_starts + self.hyperparameter_starts = hyperparameter_starts + self.seed = seed + + def _describe(self) -> dict[str, Any]: + initial_genomes = None + if self.initial_genomes is not None: + rows = np.asarray(self.initial_genomes, dtype=object) + if rows.ndim != 2: + raise ValueError( + "initial_genomes is a genome per row, a number per gene, not " + f"{self.initial_genomes!r}" + ) + initial_genomes = [ + [_number("initial_genomes", gene, plural=True) for gene in row] for row in rows + ] + if self.output not in ("standardize", "log"): + raise ValueError(f'output is "standardize" or "log", not {self.output!r}') + return { + "type": "bo", + "initial_points": _optional_whole("initial_points", self.initial_points), + "initial_genomes": initial_genomes, + "acquisition": _acquisition(self.acquisition), + "kernel": _surrogate._kernel(self.kernel), + "noise": _surrogate._noise(self.noise), + "output": self.output, + "raw_samples": _whole("raw_samples", self.raw_samples), + "acquisition_starts": _whole("acquisition_starts", self.acquisition_starts), + "hyperparameter_starts": _whole("hyperparameter_starts", self.hyperparameter_starts), + "seed": _optional_whole("seed", self.seed), + } + + class _GradientMethod(_SingleObjective): """A gradient-based method: its ``run`` takes the gradient of the fitness function.""" @@ -5261,3 +5500,5 @@ def _describe(self) -> dict[str, Any]: # the submodules use the classes above from . import indicators, math, nn, problems # noqa: E402 from . import gp # noqa: E402 +from . import model # noqa: E402 +from .model import gp as _surrogate # noqa: E402 diff --git a/python/genoxide/_genoxide.pyi b/python/genoxide/_genoxide.pyi index 74db87c3..6be5dc53 100644 --- a/python/genoxide/_genoxide.pyi +++ b/python/genoxide/_genoxide.pyi @@ -31,6 +31,28 @@ class Running: def get(self, name: str) -> str: ... def set(self, name: str, value: str) -> None: ... def reevaluate(self) -> None: ... + def model(self) -> GaussianProcess | None: ... + def acquisition(self, points: np.ndarray) -> np.ndarray: ... + +class GaussianProcess: + """A Gaussian process of ``genoxide::model::gp``, fitted from its description.""" + + def __init__(self, description: str, points: np.ndarray, values: np.ndarray) -> None: ... + def predict(self, points: np.ndarray) -> tuple[np.ndarray, np.ndarray]: ... + def predict_with_gradient( + self, point: np.ndarray + ) -> tuple[float, float, np.ndarray, np.ndarray]: ... + @property + def hyperparameters(self) -> tuple[float, list[float], float, float]: ... + @property + def log_marginal_likelihood(self) -> float: ... + @property + def jitter(self) -> float: ... + @property + def kernel(self) -> str: ... + @property + def genes(self) -> int: ... + def __len__(self) -> int: ... class Snapshot: """A population (or a front) after a generation, for a progress object: its arrays are made diff --git a/python/genoxide/model/__init__.py b/python/genoxide/model/__init__.py new file mode 100644 index 00000000..a0c2f835 --- /dev/null +++ b/python/genoxide/model/__init__.py @@ -0,0 +1,9 @@ +"""Surrogate models: cheap approximations of an expensive function, fitted to its evaluations. + +:mod:`genoxide.model.gp` has Gaussian process regression, the model of :class:`genoxide.Bo`, +which can also be fitted and queried on its own. +""" + +from . import gp + +__all__ = ["gp"] diff --git a/python/genoxide/model/gp.py b/python/genoxide/model/gp.py new file mode 100644 index 00000000..51c69c11 --- /dev/null +++ b/python/genoxide/model/gp.py @@ -0,0 +1,235 @@ +"""Gaussian process regression, fitted in Rust: the surrogate model of :class:`genoxide.Bo`, which +can also be fitted and queried on its own. + +**Unstable for one release.** The module is public from genoxide 0.13, but its API and the bits of +its fits may still change in 0.14. + +A :class:`GaussianProcess` models a function of the genes of a :class:`genoxide.Real` genome from +its values at some points, and gives the posterior mean and variance of the function anywhere, +with their gradients (Rasmussen and Williams 2006, eq. 2.25 and 2.26):: + + import numpy as np + import genoxide as gx + + # a smooth function of one gene, from 8 evaluations + x = np.arange(8.0).reshape(-1, 1) + y = np.sin(x[:, 0]) + 0.1 * x[:, 0] ** 2 + model = gx.model.gp.GaussianProcess.fit(gx.Real((0, 7), length=1), x, y) + mean, variance = model.predict([[3.5]]) + +The model has a constant mean and a kernel with a length scale per gene, "matern52" (Matérn, +ν = 5/2, the default) or "squared_exponential"; inputs are scaled to the unit cube by the genome's +bounds, and values standardized. Without noise by default, the model interpolates the values, as +suits a deterministic function; :class:`Learned` noise is for a noisy one. The hyperparameters +maximize the log marginal likelihood (eq. 2.30), with genoxide's L-BFGS-B and the gradient of eq. +5.9, from ``starts`` starts (the first from fixed values, the others random, from ``seed``): the +same fit on every platform, and the one genoxide's Rust ``model::gp`` gives. + +References: Rasmussen, C. E. and Williams, C. K. I. (2006). *Gaussian Processes for Machine +Learning.* MIT Press, ch. 2, 4 and 5. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any, Literal + +import numpy as np + +from .. import Real, _describe_setting, _genoxide, _number, _whole + +__all__ = ["GaussianProcess", "Hyperparameters", "Learned", "Kernel"] + +Kernel = Literal["matern52", "squared_exponential"] +_KERNELS = ("matern52", "squared_exponential") + + +@dataclass(frozen=True) +class Learned: + """Noise learned with the other hyperparameters, its variance at least ``min`` of the values' + variance (between 0 and 1, exclusive): for a noisy function, whose values the model shouldn't + interpolate.""" + + min: float = 1e-6 + + def _describe(self) -> dict[str, Any]: + return {"type": "learned", "min": _number("Learned.min", self.min)} + + +@dataclass(frozen=True) +class Hyperparameters: + """A Gaussian process's hyperparameters, in the units of the genes and of the values: the + constant ``mean``, a length scale per gene (infinite for a gene whose bounds are equal), the + signal variance (the function's variance far from any data) and the noise variance.""" + + mean: float + length_scales: tuple[float, ...] + signal_variance: float + noise_variance: float + + +def _noise(noise: Any) -> dict[str, Any]: + """A noise setting: a fixed variance, a fraction of the values' variance, or :class:`Learned`.""" + if isinstance(noise, Learned): + return noise._describe() + return {"type": "fixed", "variance": _number("noise", noise)} + + +def _kernel(kernel: Any) -> str: + if kernel not in _KERNELS: + raise ValueError(f'kernel is "matern52" or "squared_exponential", not {kernel!r}') + return str(kernel) + + +def _rows(name: str, points: Any, genes: int) -> np.ndarray: + """Points as a 2-D float64 array, a point per row: a 1-D array is one point.""" + rows = np.ascontiguousarray(points, dtype=np.float64) + if rows.ndim == 1: + rows = rows.reshape(1, -1) + if rows.ndim != 2 or rows.shape[1] != genes: + raise ValueError( + f"{name} are points of {genes} genes, a point per row, not of shape {np.shape(points)}" + ) + return rows + + +class GaussianProcess: + """A Gaussian process fitted to evaluations of a function, from :meth:`fit`, or the model of a + running :class:`genoxide.Bo` (:attr:`genoxide.RunningBo.model`).""" + + __slots__ = ("_native",) + + def __init__(self, native: _genoxide.GaussianProcess) -> None: + self._native = native + + @classmethod + def fit( + cls, + genome: Real, + points: Any, + values: Any, + *, + kernel: Kernel = "matern52", + noise: float | Learned = 0.0, + starts: int = 5, + seed: int = 0, + hyperparameters: Hyperparameters | None = None, + ) -> GaussianProcess: + """Fits a model to ``values`` at ``points``. + + Parameters + ---------- + genome : Real + The genes, whose bounds scale the model's inputs to the unit cube. + points : array-like of float + A point per row, a value per gene, all finite; at least one point. + values : array-like of float + A finite value per point. + kernel : "matern52" or "squared_exponential", default "matern52" + The kernel: Matérn's with ν = 5/2 (twice differentiable functions), or the squared + exponential (infinitely differentiable). + noise : float or Learned, default 0.0 + The noise variance as a fraction of the values' variance: a fixed one, at least 0 (0 + interpolates the values), or :class:`Learned` with the other hyperparameters. + starts : int, default 5 + The starts of the likelihood's maximization, at least 1. + seed : int, default 0 + The seed of the random starts, 0 to 2^64 - 1. + hyperparameters : Hyperparameters, optional + Hyperparameters to use as they are, instead of fitting them; ``noise`` is then + ignored. + + Raises + ------ + ValueError + For a wrong setting, no points, a point without a value per gene, or a point or value + that isn't finite. + """ + described = _describe_setting("genome", genome, "a Real") + if described.get("type") != "real": + raise ValueError(f"genome is a Real, not {genome!r}") + bounds = described["bounds"] + rows = _rows("points", points, len(bounds)) + array = np.ascontiguousarray(values, dtype=np.float64) + if array.ndim != 1: + raise ValueError( + f"values is a 1-D array, a value per point, not of shape {array.shape}" + ) + given = None + if hyperparameters is not None: + given = { + "mean": _number("hyperparameters.mean", hyperparameters.mean), + "length_scales": [ + _number("hyperparameters.length_scales", length, plural=True) + for length in hyperparameters.length_scales + ], + "signal_variance": _number( + "hyperparameters.signal_variance", hyperparameters.signal_variance + ), + "noise_variance": _number( + "hyperparameters.noise_variance", hyperparameters.noise_variance + ), + } + description = { + "bounds": bounds, + "kernel": _kernel(kernel), + "noise": _noise(noise), + "starts": _whole("starts", starts), + "seed": _whole("seed", seed, minimum=0, maximum=2**64 - 1), + "hyperparameters": given, + } + return cls(_genoxide.GaussianProcess(json.dumps(description), rows, array)) + + def predict(self, points: Any) -> tuple[np.ndarray, np.ndarray]: + """The posterior means and variances of the function at ``points``, a point per row (a + 1-D array is one point): two 1-D arrays, a value per point. The variances are of the + function, without the noise, and at least 0.""" + rows = _rows("points", points, self.genes) + means, variances = self._native.predict(rows) + return np.asarray(means), np.asarray(variances) + + def predict_with_gradient(self, point: Any) -> tuple[float, float, np.ndarray, np.ndarray]: + """The posterior mean and variance at ``point``, with their gradients with respect to the + genes (0 for a gene whose bounds are equal).""" + array = np.ascontiguousarray(point, dtype=np.float64) + if array.ndim != 1: + raise ValueError(f"point is a 1-D array, a value per gene, not of shape {array.shape}") + mean, variance, dmean, dvariance = self._native.predict_with_gradient(array) + return float(mean), float(variance), np.asarray(dmean), np.asarray(dvariance) + + @property + def hyperparameters(self) -> Hyperparameters: + """The hyperparameters, fitted or given.""" + mean, length_scales, signal, noise = self._native.hyperparameters + return Hyperparameters(float(mean), tuple(length_scales), float(signal), float(noise)) + + @property + def log_marginal_likelihood(self) -> float: + """The log marginal likelihood of the values at the hyperparameters (eq. 2.30).""" + return float(self._native.log_marginal_likelihood) + + @property + def jitter(self) -> float: + """The jitter added to the kernel matrix's diagonal for its factorization, in the values' + units: usually 0.""" + return float(self._native.jitter) + + @property + def kernel(self) -> Kernel: + """The kernel.""" + if self._native.kernel == "squared_exponential": + return "squared_exponential" + return "matern52" + + @property + def genes(self) -> int: + """The number of genes of a point.""" + return int(self._native.genes) + + def __len__(self) -> int: + """The number of points the model was fitted to.""" + return len(self._native) + + def __repr__(self) -> str: + return f"GaussianProcess(points={len(self)}, kernel={self.kernel!r})" diff --git a/python/src/config.rs b/python/src/config.rs index dce3538c..f0f204ce 100644 --- a/python/src/config.rs +++ b/python/src/config.rs @@ -152,6 +152,22 @@ pub enum Algorithm { initial_genome: Option>, seed: Option, }, + /// Bayesian optimization. + Bo { + /// The points of the initial design. + initial_points: Option, + /// Genomes evaluated first, in the initial design. + initial_genomes: Option>>, + acquisition: Option, + kernel: Option, + noise: Option, + output: Option, + /// The random points at which the acquisition is evaluated before its maximization. + raw_samples: Option, + acquisition_starts: Option, + hyperparameter_starts: Option, + seed: Option, + }, NelderMead { coefficients: Option, /// The size of the first simplex, as a fraction of each gene's range. @@ -592,6 +608,41 @@ pub enum GradientSource { Central, } +/// An acquisition function of Bayesian optimization. +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +pub enum Acquisition { + ExpectedImprovement, + LogExpectedImprovement, + ProbabilityOfImprovement { xi: f64 }, + UpperConfidenceBound { beta: f64 }, +} + +/// The kernel of a Gaussian process. +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum KernelName { + Matern52, + SquaredExponential, +} + +/// The noise of a Gaussian process: a fixed variance, or learned from a least one, as fractions +/// of the values' variance. +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] +pub enum NoiseConfig { + Fixed { variance: f64 }, + Learned { min: f64 }, +} + +/// What the model of Bayesian optimization fits. +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum BoOutput { + Standardize, + Log, +} + /// Nelder-Mead's coefficients: `"adaptive"` (Gao and Han's), `"standard"` (Nelder and Mead's) or /// `{reflection, expansion, contraction, shrink}`. #[derive(Clone, Copy, Debug, Deserialize)] diff --git a/python/src/control.rs b/python/src/control.rs index bacad48e..1a6284e5 100644 --- a/python/src/control.rs +++ b/python/src/control.rs @@ -9,6 +9,7 @@ use crate::config; use crate::errors::setting; +use crate::model::PyGaussianProcess; use crate::operators::AnySelect; use crate::run::{de_control, de_strategy}; use genoxide::algorithm::islands::Migrate; @@ -17,6 +18,8 @@ use genoxide::genome::Representation; use genoxide::neat::Neat; use genoxide::operator::{Crossover, Mutate}; use genoxide::prelude::*; +use numpy::ndarray::ArrayView2; +use numpy::{PyArray1, PyReadonlyArray2}; use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::*; use serde::de::DeserializeOwned; @@ -32,6 +35,19 @@ pub trait Settings: Send + Sync + 'static { /// Changes a setting to `value`, JSON; nothing changes on errors. fn set(&self, algorithm: &mut A, name: &str, value: &str) -> Result<()>; + + /// The surrogate model that chose the last point, and its number of genes, for the + /// algorithms that have one: Bayesian optimization's. None by default. + fn model(&self, algorithm: &A) -> Option { + let _ = algorithm; + None + } + + /// The acquisition function at `points`, a point per row, for the algorithms that have one. + fn acquisition(&self, algorithm: &A, points: ArrayView2<'_, f64>) -> Result> { + let _ = (algorithm, points); + Err("the algorithm has no acquisition function".to_string()) + } } fn unknown(name: &str) -> String { @@ -432,6 +448,76 @@ where } } +/// Bayesian optimization's acquisition function, which can change, its model and its +/// acquisition's values, for a plot. +pub struct BoSettings; + +impl Settings for BoSettings { + fn get(&self, bo: &Bo, name: &str) -> Result { + match name { + "acquisition" => Ok(match bo.acquisition() { + bo::Acquisition::ExpectedImprovement => json!({"type": "expected_improvement"}), + bo::Acquisition::LogExpectedImprovement => { + json!({"type": "log_expected_improvement"}) + } + bo::Acquisition::ProbabilityOfImprovement { xi } => { + json!({"type": "probability_of_improvement", "xi": xi}) + } + bo::Acquisition::UpperConfidenceBound { beta } => { + json!({"type": "upper_confidence_bound", "beta": beta}) + } + acquisition => { + return Err(format!( + "an acquisition the package doesn't know: {acquisition:?}" + )); + } + }), + "initial_points" => Ok(json!(bo.initial_points())), + _ => Err(unknown(name)), + } + } + + fn set(&self, bo: &mut Bo, name: &str, value: &str) -> Result<()> { + match name { + "acquisition" => { + let acquisition: config::Acquisition = parse(name, value)?; + setting(bo.set_acquisition(crate::model::acquisition(acquisition))) + } + _ => Err(unknown(name)), + } + } + + fn model(&self, bo: &Bo) -> Option { + let genes = bo.real().genome_len(); + bo.model() + .map(|model| PyGaussianProcess::new(model.clone(), genes)) + } + + fn acquisition(&self, bo: &Bo, points: ArrayView2<'_, f64>) -> Result> { + let genes = bo.real().genome_len(); + if points.ncols() != genes { + return Err(format!( + "points is a point per row, a value per gene of the genome's {genes}, not of shape ({}, {})", + points.nrows(), + points.ncols() + )); + } + let mut point = vec![0.0; genes]; + let mut values = Vec::with_capacity(points.nrows()); + for row in points.rows() { + for (x, &value) in point.iter_mut().zip(row.iter()) { + *x = value; + } + match bo.acquisition_at(&point) { + Some(value) => values.push(value), + None => return Err("no model yet: the first point after the initial design has its model" + .to_string()), + } + } + Ok(values) + } +} + /// Where the algorithm is while the control runs, and its settings. pub struct Slot { algorithm: Mutex>, @@ -482,6 +568,8 @@ trait AnySlot: Send + Sync { fn get(&self, name: &str) -> PyResult; fn set(&self, name: &str, value: &str) -> PyResult<()>; fn reevaluate(&self) -> PyResult<()>; + fn model(&self) -> PyResult>; + fn acquisition(&self, points: ArrayView2<'_, f64>) -> PyResult>; } impl AnySlot for Slot @@ -501,6 +589,14 @@ where fn reevaluate(&self) -> PyResult<()> { self.with(|algorithm, _| setting(algorithm.reevaluate())) } + + fn model(&self) -> PyResult> { + self.with(|algorithm, settings| Ok(settings.model(algorithm))) + } + + fn acquisition(&self, points: ArrayView2<'_, f64>) -> PyResult> { + self.with(|algorithm, settings| settings.acquisition(algorithm, points)) + } } /// The running algorithm, for a control: valid during the control's call only. The Python @@ -536,6 +632,21 @@ impl Running { fn reevaluate(&self) -> PyResult<()> { self.slot.reevaluate() } + + /// The surrogate model that chose the last point, or None. + fn model(&self) -> PyResult> { + self.slot.model() + } + + /// The acquisition function at `points`, a point per row. + fn acquisition<'py>( + &self, + py: Python<'py>, + points: PyReadonlyArray2<'py, f64>, + ) -> PyResult>> { + let values = self.slot.acquisition(points.as_array())?; + Ok(PyArray1::from_vec(py, values)) + } } /// A continuation: the wrapped method's settings, and the current stage, read-only. diff --git a/python/src/lib.rs b/python/src/lib.rs index c3a4421a..dd8f94a2 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -11,6 +11,7 @@ mod errors; mod fitness; mod genes; mod indicators; +mod model; mod neat; mod networks; mod operators; @@ -29,6 +30,7 @@ fn _genoxide(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_function(wrap_pyfunction!(run::run, module)?)?; module.add_class::()?; module.add_class::()?; + module.add_class::()?; module.add_function(wrap_pyfunction!(run::das_dennis, module)?)?; module.add_function(wrap_pyfunction!(problems::problem_info, module)?)?; module.add_function(wrap_pyfunction!(problems::evaluate, module)?)?; diff --git a/python/src/model.rs b/python/src/model.rs new file mode 100644 index 00000000..9cd2cb07 --- /dev/null +++ b/python/src/model.rs @@ -0,0 +1,234 @@ +//! The Gaussian process of `genoxide::model::gp`, for `gx.model.gp`: fitted in Rust from the +//! description the Python package makes, and queried on numpy arrays. + +use crate::config::{Acquisition, KernelName, NoiseConfig}; +use crate::errors::{genome_setting, setting}; +use genoxide::algorithm::bo; +use genoxide::genome::{Real, Reals}; +use genoxide::model::gp::{GaussianProcess, Hyperparameters, Kernel, Noise}; +use numpy::{PyArray1, PyReadonlyArray1, PyReadonlyArray2}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use serde::Deserialize; + +/// The kernel of a name. +pub fn kernel(name: KernelName) -> Kernel { + match name { + KernelName::Matern52 => Kernel::Matern52, + KernelName::SquaredExponential => Kernel::SquaredExponential, + } +} + +/// The noise of a description. +pub fn noise(noise: NoiseConfig) -> Noise { + match noise { + NoiseConfig::Fixed { variance } => Noise::Fixed(variance), + NoiseConfig::Learned { min } => Noise::Learned { min }, + } +} + +/// The acquisition function of a description. +pub fn acquisition(acquisition: Acquisition) -> bo::Acquisition { + match acquisition { + Acquisition::ExpectedImprovement => bo::Acquisition::ExpectedImprovement, + Acquisition::LogExpectedImprovement => bo::Acquisition::LogExpectedImprovement, + Acquisition::ProbabilityOfImprovement { xi } => { + bo::Acquisition::ProbabilityOfImprovement { xi } + } + Acquisition::UpperConfidenceBound { beta } => { + bo::Acquisition::UpperConfidenceBound { beta } + } + } +} + +/// A fit, as `gx.model.gp.GaussianProcess.fit` describes it. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct Description { + bounds: Vec<(f64, f64)>, + kernel: KernelName, + noise: NoiseConfig, + starts: usize, + seed: u64, + hyperparameters: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct GivenHyperparameters { + mean: f64, + length_scales: Vec, + signal_variance: f64, + noise_variance: f64, +} + +// two arrays of a value per point +type Columns<'py> = (Bound<'py, PyArray1>, Bound<'py, PyArray1>); + +/// A fitted Gaussian process. +#[pyclass(frozen, module = "genoxide._genoxide", name = "GaussianProcess")] +pub struct PyGaussianProcess { + model: GaussianProcess, + genes: usize, +} + +impl PyGaussianProcess { + /// A fitted model of `genes` genes, for Python. + pub fn new(model: GaussianProcess, genes: usize) -> Self { + Self { model, genes } + } + + // a ValueError unless `point` has a value per gene + fn check(&self, what: &str, len: usize) -> PyResult<()> { + if len == self.genes { + Ok(()) + } else { + Err(PyValueError::new_err(format!( + "{what} has {len} values, not one per gene of the model's {}", + self.genes + ))) + } + } +} + +#[pymethods] +impl PyGaussianProcess { + /// Fits a model to `values` at `points`, a point per row, as `description` (JSON) says. + #[new] + fn fit( + py: Python<'_>, + description: &str, + points: PyReadonlyArray2<'_, f64>, + values: PyReadonlyArray1<'_, f64>, + ) -> PyResult { + let description: Description = serde_json::from_str(description) + .map_err(|error| PyValueError::new_err(format!("invalid model: {error}")))?; + let real = genome_setting( + Real::new(description.bounds.iter().map(|&(low, high)| low..=high)), + "Real", + ) + .map_err(PyValueError::new_err)?; + let genes = description.bounds.len(); + let points = points.as_array(); + if points.ncols() != genes { + return Err(PyValueError::new_err(format!( + "points is a point per row, a value per gene of the genome's {genes}, not of shape \ + ({}, {})", + points.nrows(), + points.ncols() + ))); + } + let points: Vec = points + .rows() + .into_iter() + .map(|row| row.iter().copied().collect()) + .collect(); + let values: Vec = values.as_array().iter().copied().collect(); + let mut builder = GaussianProcess::builder(real) + .kernel(kernel(description.kernel)) + .noise(noise(description.noise)) + .starts(description.starts) + .seed(description.seed); + if let Some(h) = description.hyperparameters { + builder = builder.hyperparameters(Hyperparameters::new( + h.mean, + h.length_scales, + h.signal_variance, + h.noise_variance, + )); + } + let model = py + .detach(|| setting(builder.fit(&points, &values))) + .map_err(PyValueError::new_err)?; + Ok(Self::new(model, genes)) + } + + /// The posterior means and variances at `points`, a point per row. + fn predict<'py>( + &self, + py: Python<'py>, + points: PyReadonlyArray2<'py, f64>, + ) -> PyResult> { + let points = points.as_array(); + self.check("a point", points.ncols())?; + let mut point = vec![0.0; self.genes]; + let (mut means, mut variances) = (Vec::new(), Vec::new()); + for row in points.rows() { + for (x, &value) in point.iter_mut().zip(row.iter()) { + *x = value; + } + let prediction = self.model.predict(&point); + means.push(prediction.mean()); + variances.push(prediction.variance()); + } + Ok(( + PyArray1::from_vec(py, means), + PyArray1::from_vec(py, variances), + )) + } + + /// The posterior mean and variance at `point`, with their gradients. + #[allow(clippy::type_complexity)] + fn predict_with_gradient<'py>( + &self, + py: Python<'py>, + point: PyReadonlyArray1<'py, f64>, + ) -> PyResult<( + f64, + f64, + Bound<'py, PyArray1>, + Bound<'py, PyArray1>, + )> { + let point: Vec = point.as_array().iter().copied().collect(); + self.check("point", point.len())?; + let (mut dmean, mut dvariance) = (vec![0.0; self.genes], vec![0.0; self.genes]); + let prediction = self + .model + .predict_with_gradient(&point, &mut dmean, &mut dvariance); + Ok(( + prediction.mean(), + prediction.variance(), + PyArray1::from_vec(py, dmean), + PyArray1::from_vec(py, dvariance), + )) + } + + /// The hyperparameters: the mean, the length scales, the signal and noise variances. + #[getter] + fn hyperparameters(&self) -> (f64, Vec, f64, f64) { + let h = self.model.hyperparameters(); + ( + h.mean(), + h.length_scales().to_vec(), + h.signal_variance(), + h.noise_variance(), + ) + } + + #[getter] + fn log_marginal_likelihood(&self) -> f64 { + self.model.log_marginal_likelihood() + } + + #[getter] + fn jitter(&self) -> f64 { + self.model.jitter() + } + + #[getter] + fn kernel(&self) -> &'static str { + match self.model.kernel() { + Kernel::SquaredExponential => "squared_exponential", + _ => "matern52", + } + } + + #[getter] + fn genes(&self) -> usize { + self.genes + } + + fn __len__(&self) -> usize { + self.model.len() + } +} diff --git a/python/src/run.rs b/python/src/run.rs index 709ba263..23d03e38 100644 --- a/python/src/run.rs +++ b/python/src/run.rs @@ -3,8 +3,8 @@ use crate::checkpoint::Checkpoints; use crate::config; use crate::control::{ - CmaesSettings, ContinuationSettings, DeSettings, EsSettings, FirstOrderSettings, GaSettings, - IslandsSettings, LbfgsbSettings, LocalSearchSettings, MmaSettings, NeatSettings, + BoSettings, CmaesSettings, ContinuationSettings, DeSettings, EsSettings, FirstOrderSettings, + GaSettings, IslandsSettings, LbfgsbSettings, LocalSearchSettings, MmaSettings, NeatSettings, NelderMeadSettings, OpenEsSettings, PsoSettings, Running, Settings, Slot, }; use crate::errors::{genome_setting, setting}; @@ -850,6 +850,62 @@ fn real_algorithm<'py>( )?; generational(py, open_es, OpenEsSettings, context) } + config::Algorithm::Bo { + initial_points, + initial_genomes, + acquisition, + kernel, + noise, + output, + raw_samples, + acquisition_starts, + hyperparameter_starts, + seed, + } => { + let mut builder = Bo::builder(real).objective(context.single_objective()?); + if let Some(points) = initial_points { + builder = builder.initial_points(points); + } + if let Some(genomes) = initial_genomes { + builder = builder.initial_genomes(genomes.into_iter().map(Reals::from)); + } + if let Some(acquisition) = acquisition { + builder = builder.acquisition(crate::model::acquisition(acquisition)); + } + if let Some(kernel) = kernel { + builder = builder.kernel(crate::model::kernel(kernel)); + } + if let Some(noise) = noise { + builder = builder.noise(crate::model::noise(noise)); + } + if let Some(output) = output { + builder = builder.output(match output { + config::BoOutput::Standardize => bo::Output::Standardize, + config::BoOutput::Log => bo::Output::Log, + }); + } + if let Some(samples) = raw_samples { + builder = builder.raw_samples(samples); + } + if let Some(starts) = acquisition_starts { + builder = builder.acquisition_starts(starts); + } + if let Some(starts) = hyperparameter_starts { + builder = builder.hyperparameter_starts(starts); + } + if let Some(seed) = seed { + builder = builder.seed(seed); + } + // the only genomes the builder checks are the initial ones + let bo = setting(builder.build().map_err(|error| match error { + genoxide::Error::InvalidGenome { reason } => genoxide::Error::InvalidSetting { + setting: "initial_genomes", + reason, + }, + error => error, + }))?; + generational(py, bo, BoSettings, context) + } config::Algorithm::NelderMead { coefficients, initial_step, @@ -1283,6 +1339,7 @@ where config::Algorithm::NelderMead { .. } => { Err("NelderMead needs a Real genome".to_string().into()) } + config::Algorithm::Bo { .. } => Err("Bo needs a Real genome".to_string().into()), config::Algorithm::Lbfgsb { .. } => Err("Lbfgsb needs a Real genome".to_string().into()), config::Algorithm::Mma { .. } => Err("Mma needs a Real genome".to_string().into()), config::Algorithm::Continuation { .. } => { diff --git a/python/tests/test_bo.py b/python/tests/test_bo.py new file mode 100644 index 00000000..e7f2d96e --- /dev/null +++ b/python/tests/test_bo.py @@ -0,0 +1,273 @@ +"""Bayesian optimization and its Gaussian process: the minima of the classic problems, the same runs +as in Rust, the running algorithm's model, checkpoints and the settings.""" + +import math + +import numpy as np +import pytest + +import genoxide as gx + +BRANIN_MINIMUM = 5 / (4 * math.pi) + + +def test_branin_in_tens_of_evaluations(): + problem = gx.problems.Branin() + result = gx.Bo(problem.genome, objective="minimize", seed=1).run(problem, evaluations=40) + assert result.best_fitness - BRANIN_MINIMUM < 1e-3 + assert result.evaluations == 40 + # the design is generation 0, then a point per generation + assert result.generations == 40 - 6 + + +def test_a_run_as_in_rust(): + # tests/bo.rs has the same run in Rust, with the same best value, to the bit + problem = gx.problems.Branin() + bo = gx.Bo(problem.genome, objective="minimize", seed=1) + for function, parallel in [(problem, False), (lambda x: problem(x), False), (problem, True)]: + result = bo.run(function, evaluations=30, parallel=parallel) + assert result.best_fitness == 0.39798370755715595 + assert result.best_genome.tolist() == [9.424508858501198, 2.484571084777484] + + +def test_the_log_transform_on_goldstein_price(): + problem = gx.problems.GoldsteinPrice() + bo = gx.Bo(problem.genome, output="log", objective="minimize", seed=1) + result = bo.run(problem, target=3.001, evaluations=70) + assert result.stop_reason == "target" + + +def test_maximizing_mirrors_minimizing(): + problem = gx.problems.Branin() + minimized = gx.Bo(problem.genome, objective="minimize", seed=4).run(problem, evaluations=15) + maximized = gx.Bo(problem.genome, objective="maximize", seed=4).run( + lambda x: -problem(x), evaluations=15 + ) + assert np.array_equal(maximized.best_genome, minimized.best_genome) + assert maximized.best_fitness == -minimized.best_fitness + + +def test_every_acquisition_and_the_other_kernel(): + problem = gx.problems.Branin() + for acquisition, kernel in [ + ("ei", "matern52"), + (gx.ProbabilityOfImprovement(0.01), "matern52"), + (gx.UpperConfidenceBound(4.0), "matern52"), + ("log-ei", "squared_exponential"), + ]: + bo = gx.Bo( + problem.genome, acquisition=acquisition, kernel=kernel, objective="minimize", seed=1 + ) + result = bo.run(problem, evaluations=40) + assert result.best_fitness < BRANIN_MINIMUM + 0.05, (acquisition, kernel) + + +def test_initial_genomes_come_first_and_points_are_never_asked_twice(): + genome = gx.Real((0, 1), length=3) + given = np.array([[0.5, 0.5, 0.5], [0.1, 0.9, 0.2]]) + seen = [] + + def sphere(x): + seen.append(x.copy()) + return float(np.sum((x - 0.3) ** 2)) + + gx.Bo(genome, initial_genomes=given, objective="minimize", seed=1).run( + sphere, evaluations=20 + ) + assert len(seen) == 20 + assert np.array_equal(seen[0], given[0]) and np.array_equal(seen[1], given[1]) + assert len({tuple(x) for x in seen}) == 20 + + +def test_invalid_points_enter_the_model_at_the_worst_value(): + problem = gx.problems.Branin() + + def partial(x): + return None if x[0] + x[1] > 14 else problem(x) + + result = gx.Bo(problem.genome, objective="minimize", seed=1).run(partial, evaluations=45) + assert result.best_fitness < BRANIN_MINIMUM + 1e-2 + + +def test_the_running_algorithm_gives_its_model_and_acquisition(): + problem = gx.problems.Branin() + seen = {} + + def control(algorithm, progress): + if progress.generation == 0: + assert algorithm.model is None + assert algorithm.initial_points == 6 + with pytest.raises(ValueError, match="no model"): + algorithm.acquisition_at([0.0, 5.0]) + return + model = algorithm.model + assert len(model) == 5 + progress.generation + newest = progress.population[-1] + values = algorithm.acquisition_at(np.array([newest, [0.0, 5.0]])) + assert values.shape == (2,) and np.all(np.isfinite(values)) + # the newest point maximized the acquisition + grid = np.array([[x1, x2] for x1 in np.linspace(-5, 10, 16) for x2 in np.linspace(0, 15, 16)]) + assert values[0] >= algorithm.acquisition_at(grid).max() - 1e-9 + seen["acquisition"] = algorithm.acquisition + algorithm.acquisition = gx.UpperConfidenceBound(9.0 / (1 + progress.generation)) + with pytest.raises(ValueError, match="beta"): + algorithm.acquisition = gx.UpperConfidenceBound(-1.0) + + result = gx.Bo(problem.genome, objective="minimize", seed=1).run( + problem, evaluations=20, control=control + ) + assert seen["acquisition"] == gx.UpperConfidenceBound(9.0 / 14) + assert result.best_fitness < 1.0 + + +def test_reevaluation_asks_every_point_again(): + problem = gx.problems.Branin() + calls = [] + shift = {"value": 0.0} + + def shifted(x): + calls.append(x.copy()) + return problem(x) + shift["value"] + + def control(algorithm, progress): + if progress.generation == 3 and shift["value"] == 0.0: + shift["value"] = 10.0 + algorithm.reevaluate() + + result = gx.Bo(problem.genome, objective="minimize", seed=2).run( + shifted, generations=5, control=control + ) + # 6 points, 3 more, all 9 again, then 2 more + assert len(calls) == 6 + 3 + 9 + 2 + assert result.best_fitness > 10 + + +def test_a_checkpoint_resumes_the_run(tmp_path): + problem = gx.problems.Branin() + bo = gx.Bo(problem.genome, objective="minimize", seed=3) + whole = bo.run(problem, evaluations=20) + path = tmp_path / "bo.ckpt" + bo.run(problem, evaluations=12, checkpoint=path, checkpoint_every=2) + resumed = bo.run(problem, evaluations=20, resume=path) + assert resumed.best_fitness == whole.best_fitness + assert np.array_equal(resumed.best_genome, whole.best_genome) + assert resumed.evaluations == whole.evaluations + + +@pytest.mark.parametrize( + "settings, message", + [ + ({"acquisition": "ucb"}, "acquisition"), + ({"acquisition": gx.UpperConfidenceBound(-1.0)}, "beta"), + ({"acquisition": gx.ProbabilityOfImprovement(math.nan)}, "xi"), + ({"kernel": "rbf"}, "kernel"), + ({"output": "rank"}, "output"), + ({"noise": -1.0}, "noise"), + ({"noise": gx.model.gp.Learned(0.0)}, "noise"), + ({"initial_points": 0}, "initial_points"), + ({"initial_genomes": [0.5, 0.5]}, "initial_genomes"), + ({"initial_genomes": [[0.5, 0.5], [0.5, 0.5]]}, "initial_genomes"), + ({"initial_genomes": [[2.0, 0.5]]}, "initial_genomes"), + ({"raw_samples": 0}, "raw_samples"), + ({"acquisition_starts": 2000}, "acquisition_starts"), + ({"hyperparameter_starts": 0}, "hyperparameter_starts"), + ], +) +def test_wrong_settings_are_value_errors(settings, message): + bo = gx.Bo(gx.Real((0, 1), length=2), **settings) + with pytest.raises(ValueError, match=message): + bo.run(lambda x: float(np.sum(x)), evaluations=10) + + +def test_a_bo_needs_a_real_genome(): + with pytest.raises(ValueError, match="Real"): + gx.Bo(gx.Binary(4)).run(lambda x: float(x.sum()), evaluations=10) + + +# ---- the Gaussian process ------------------------------------------------------------------------ + + +def test_two_points_are_the_formulas_worked_out_by_hand(): + # Rasmussen and Williams (2006): the mean m + k*' K^-1 (y - m) (eq. 2.38), the variance + # s_f^2 - k*' K^-1 k* (eq. 2.26) and the log marginal likelihood (eq. 2.30), the squared + # exponential kernel in a gene of [-2, 6] + x, y = np.array([0.5, 2.0]), np.array([1.0, 4.0]) + length, signal, noise, mean = 2.0, 3.0, 0.1, 1.5 + model = gx.model.gp.GaussianProcess.fit( + gx.Real((-2, 6), length=1), + x.reshape(-1, 1), + y, + kernel="squared_exponential", + hyperparameters=gx.model.gp.Hyperparameters(mean, (length,), signal, noise), + ) + + def k(a, b): + return signal * math.exp(-0.5 * ((a - b) / length) ** 2) + + covariance = np.array([[k(a, b) for b in x] for a in x]) + noise * np.eye(2) + alpha = np.linalg.solve(covariance, y - mean) + for at in [-2.0, 1.0, 1.7, 6.0]: + kstar = np.array([k(at, b) for b in x]) + expected_mean = mean + kstar @ alpha + expected_variance = signal - kstar @ np.linalg.solve(covariance, kstar) + means, variances = model.predict([[at]]) + assert means[0] == pytest.approx(expected_mean, rel=1e-12) + assert variances[0] == pytest.approx(expected_variance, rel=1e-11) + log_likelihood = ( + -0.5 * (y - mean) @ alpha + - 0.5 * math.log(np.linalg.det(covariance)) + - math.log(2 * math.pi) + ) + assert model.log_marginal_likelihood == pytest.approx(log_likelihood, rel=1e-12) + h = model.hyperparameters + assert (h.mean, h.length_scales[0], h.signal_variance, h.noise_variance) == pytest.approx( + (mean, length, signal, noise), rel=1e-15 + ) + assert model.kernel == "squared_exponential" and len(model) == 2 and model.genes == 1 + + +def test_a_fitted_model_interpolates_and_its_gradients_match_differences(): + genome = gx.Real([(-1, 2), (0, 3)]) + rng = np.random.default_rng(1) + points = rng.uniform([-1, 0], [2, 3], size=(15, 2)) + values = np.sin(2 * points[:, 0]) + 0.5 * points[:, 1] ** 2 + + model = gx.model.gp.GaussianProcess.fit(genome, points, values) + means, variances = model.predict(points) + assert np.allclose(means, values, atol=1e-6) + assert np.all(variances < 1e-8 * model.hyperparameters.signal_variance) + at = np.array([0.3, 1.1]) + mean, variance, dmean, dvariance = model.predict_with_gradient(at) + assert (mean, variance) == (model.predict(at)[0][0], model.predict(at)[1][0]) + for i in range(2): + h = np.zeros(2) + h[i] = 1e-6 + plus, minus = model.predict(at + h), model.predict(at - h) + assert dmean[i] == pytest.approx((plus[0][0] - minus[0][0]) / 2e-6, rel=1e-6, abs=1e-6) + assert dvariance[i] == pytest.approx((plus[1][0] - minus[1][0]) / 2e-6, abs=1e-6) + # a seed repeats the fit; learned noise is another model + again = gx.model.gp.GaussianProcess.fit(genome, points, values) + assert again.hyperparameters == model.hyperparameters + noisy = gx.model.gp.GaussianProcess.fit(genome, points, values, noise=gx.model.gp.Learned()) + assert noisy.hyperparameters.noise_variance > 0 + + +@pytest.mark.parametrize( + "arguments, message", + [ + ({"points": np.zeros((0, 2)), "values": np.zeros(0)}, "point"), + ({"points": np.zeros((3, 3)), "values": np.zeros(3)}, "points"), + ({"points": np.zeros((3, 2)), "values": np.zeros(2)}, "value"), + ({"points": np.zeros((2, 2)), "values": [0.0, math.inf]}, "value"), + ({"kernel": "rbf"}, "kernel"), + ({"noise": -1.0}, "noise"), + ({"starts": 0}, "starts"), + ], +) +def test_wrong_model_settings_are_value_errors(arguments, message): + arguments = {"points": [[0.1, 0.2], [0.5, 0.5]], "values": [1.0, 2.0], **arguments} + with pytest.raises(ValueError, match=message): + gx.model.gp.GaussianProcess.fit(gx.Real((0, 1), length=2), **arguments) + model = gx.model.gp.GaussianProcess.fit(gx.Real((0, 1), length=2), [[0.1, 0.2]], [1.0]) + with pytest.raises(ValueError, match="genes"): + model.predict([[0.5]]) diff --git a/python/tests/test_docs.py b/python/tests/test_docs.py index 2fc600fe..1bb364a1 100644 --- a/python/tests/test_docs.py +++ b/python/tests/test_docs.py @@ -43,7 +43,7 @@ def test_the_module_docstring_example_runs(): def test_the_submodule_docstring_examples_run(): modules = (gx.problems, gx.problems.cec2006, gx.problems.engineering, gx.neat) modules += (gx.gp, gx.gp.regression, gx.gp.regression.problems, gx.gp.boolean) - for module in modules + (gx.problems.multi_engineering, gx.indicators): + for module in modules + (gx.problems.multi_engineering, gx.indicators, gx.model.gp): # later examples use what earlier ones define namespace = {} for example in docstring_examples(module): diff --git a/tests/bo.rs b/tests/bo.rs index 97a9ef08..5d808fc1 100644 --- a/tests/bo.rs +++ b/tests/bo.rs @@ -153,6 +153,20 @@ fn reproducible_sequentially_and_in_parallel() { assert_ne!(run(branin_bo(6), false, 20).0, genomes); } +#[test] +fn the_python_package_gives_the_same_run() { + // python/tests/test_bo.py has the same run, and the bayesian_optimization example prints it + let outcome = Engine::new(branin_bo(1), Branin) + .stop_when(Stop::evaluations(30)) + .run() + .unwrap(); + assert_eq!(outcome.best_fitness().score(), Some(0.39798370755715595)); + assert_eq!( + outcome.best_genome()[..], + [9.424508858501198, 2.484571084777484] + ); +} + #[test] fn no_point_is_asked_twice() { let (genomes, _) = run(branin_bo(2), false, 40); From 9b5cc4d8cb34e717ed9dbcad75a22bd8a4c9152d Mon Sep 17 00:00:00 2001 From: tachsin Date: Fri, 2 Oct 2026 12:42:46 +0300 Subject: [PATCH 10/13] feat(cli): Bayesian optimization, type = "bo" The genoxide program runs Bo with every builder setting: the acquisition as "log-ei", "ei", { type = "pi", xi } or { type = "ucb", beta }, the kernel, the noise as a number or { learned = }, the output transform and the starts of both maximizations. Checkpoints resume a run as it would have gone on. --- docs/cli.md | 2 + src/bin/genoxide/config.rs | 70 +++++++++++++++++++++++ src/bin/genoxide/run.rs | 65 +++++++++++++++++++++ tests/cli.rs | 114 +++++++++++++++++++++++++++++++++++++ 4 files changed, 251 insertions(+) diff --git a/docs/cli.md b/docs/cli.md index ab44dac5..2a0b53c0 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -148,6 +148,7 @@ Set exactly one of `command` and `builtin`. | `"pso"` | real | `population_size` | `ring` (off: the whole swarm) | | `"local-search"` | any | `neighbor` (a mutation) | `neighbors` (1), `acceptance` (`not-worse`), `restart` (off) | | `"nelder-mead"` | real | | `coefficients` (`"adaptive"`), `initial_step` (0.1), `initial_step_absolute`, `tolerance` (1e-9), `restarts` (none), `speculative` (false) | +| `"bo"` | real | | `initial_points` (2(n + 1)), `acquisition` (`"log-ei"`), `kernel` (`"matern52"`), `noise` (0), `output` (`"standardize"`), `raw_samples` (1000), `acquisition_starts` (10), `hyperparameter_starts` (5) | | `"lbfgsb"` | real | | `memory` (10), `gradients` (`"auto"`), `difference_step`, `gradient_tolerance` (1e-5), `function_tolerance` (2.220446049250313e-9), `max_line_search` (20), `restarts` (none) | | `"mma"` | real | `fitness.gradient = true` | `method` (`"mma"`), `asymptote_initial` (0.5), `asymptote_decrease` (0.7), `asymptote_increase` (1.2), `move_limit` (0.5), `constraint_cost` (1000), `kkt_tolerance` (1e-9), `step_tolerance` (1e-10), `restoration` (true), `parallel_sums` (false) | | `"first-order"` | real | | `step` (`{ type = "adam", learning_rate = 0.001 }`), `gradients` (`"auto"`), `difference_step`, `gradient_tolerance` (1e-6), `step_tolerance` (1e-12), `restarts` (none) | @@ -161,6 +162,7 @@ Set exactly one of `command` and `builtin`. - `de`: SHADE's settings by default, with genoxide's restarts. `strategy` builds the mutant vectors: `"rand1"`, `"best1"`, `{ p, archive }` (current-to-pbest/1, `pbest` among the best `p` of the population, 0 < p ≤ 1, and an archive of `archive` times the population, 0 or more) or `{ max_p, archive }` (the same with a random `p` per trial up to `max_p`, as in SHADE). `control` gives F and CR: `{ f, cr }` (fixed, 0 < f ≤ 2, 0 ≤ cr ≤ 1), `{ min_f, max_f, cr }` (a random F per trial, 0 < min_f ≤ max_f ≤ 2), `{ c }` (JADE's adaptation, 0 < c ≤ 1) or `{ memory }` (SHADE's, 1 to 2^24). `restarts` is `"never"` or `{ tolerance, patience }`: all but the best are replaced when every gene's values are within `tolerance` of its range of each other and the scores within `tolerance` relative to the best (0 or more), or after `patience` generations without a better best (at least 1). With `l_shade`, they replace L-SHADE's settings, whose restarts are `"never"`. - `cmaes`: n is the number of genes whose bounds differ. `restarts` is `"never"` (a converged run goes on sampling around its point), `"ipop"`, `"bipop"` or `"stop"`: the run ends when it converges, with the stop reason `"converged"`, which saves most of a budget on smooth problems without constraints; on flat or quantized fitness and with constraints, a run can still improve after it converges. `initial_step` is the initial step size as a fraction of each gene's range, greater than 0 and at most 1. `covariance` is `"full"`, which learns the correlations between genes (O(n²) per sample: up to a few hundred genes), or `"diagonal"`, sep-CMA-ES, which learns only each gene's variance (O(n) per sample: for separable problems and thousands of genes). - `nelder-mead`: the Nelder-Mead simplex method, a local method for a few genes (up to about 10, more with the adaptive coefficients). It starts from a random point, and ends the run by itself when it has converged, with the stop reason `"converged"`; a `[stop]` condition is still needed. `coefficients` is `"adaptive"` (Gao and Han's, for the number of genes whose bounds differ), `"standard"` (Nelder and Mead's: reflection 1, expansion 2, contraction 0.5, shrink 0.5) or `{ reflection, expansion, contraction, shrink }`, with reflection greater than 0, expansion greater than 1 and than reflection, and contraction and shrink greater than 0 and less than 1. `initial_step` is the size of the first simplex as a fraction of each gene's range, greater than 0 and at most 1; `initial_step_absolute` sets it as a distance instead, the same in every gene (positive), for a wide box around an unbounded problem; give one or neither. `tolerance` is the simplex size at which a run has converged, as a fraction of the initial step, greater than 0 and less than 1. A trial point outside the bounds is mirrored back in at the bound it crossed. `restarts = ` starts again from a random point that many times (at least 1) after converging, and the result is the best of all the runs. A generation is one round of evaluations: the first simplex, one trial point, or a shrink. `speculative = true` evaluates the reflection, the expansion and both contractions in one round, so that up to four workers evaluate at the same time. +- `bo`: Bayesian optimization, for expensive functions, where tens to a few hundred evaluations must do. Generation 0 evaluates an initial design of `initial_points` (at least 1; 2(n + 1) for the n genes whose bounds differ by default) from a Latin hypercube; each later generation fits a Gaussian process to every evaluation and evaluates the one point that maximizes the acquisition function, so a single worker does: more workers only share the design. `acquisition` is `"log-ei"` (the logarithm of the expected improvement, computed so that it keeps a gradient where the expected improvement underflows), `"ei"`, `{ type = "pi", xi }` (the probability of improving by more than `xi`, 0 or more) or `{ type = "ucb", beta }` (the confidence bound with `beta` 0 or more). `kernel` is `"matern52"` (Matérn's with ν = 5/2) or `"squared-exponential"`, with a length scale per gene. `noise` is the model's noise variance as a fraction of the values' variance: a number, 0 or more (0 interpolates the values, as suits a deterministic program), or `{ learned = }`, learned with the other hyperparameters from a least variance between 0 and 1. `output = "log"` models the logarithm of the values' distance above the best, for objectives that span orders of magnitude. The acquisition is evaluated at `raw_samples` random points (at least 1), then maximized by L-BFGS-B from the best `acquisition_starts` of them (at least 1, at most `raw_samples`) and from the best point so far; the model's likelihood from `hyperparameter_starts` starts (at least 1). A point is never evaluated twice. An invalid fitness enters the model at the worst value of the others. - `lbfgsb`: L-BFGS-B, the limited-memory BFGS method with bounds, a local method for smooth functions from a few genes to millions. It starts from a random point and ends the run by itself when it has converged, with the stop reason `"converged"`; a `[stop]` condition is still needed. `gradients` is `"auto"` (the program's gradient with `fitness.gradient = true`, forward differences otherwise), `"supplied"` (needs `fitness.gradient = true`), `"forward"` or `"central"` (finite differences in any case: n or 2n more points per gradient, evaluated by the workers together with the point). `difference_step` is the relative step of finite differences, above 0, with `"forward"` or `"central"` (default √ε for forward, ε^(1/3) for central). `memory` is the number of correction pairs kept, at least 1 (3 to 20 is usual). A run has converged when the largest component of the projected gradient is at most `gradient_tolerance` (absolute, 0 or more; forward differences rarely meet much less than 1e-7 of the function's scale), or a step lowers the value by at most `function_tolerance` times max(|f|, 1) (0 or more), or no step lowers it. `max_line_search` is the most trial steps of a line search, at least 1. `restarts = ` starts again from a random point that many times (at least 1) after converging. - `mma`: Svanberg's method of moving asymptotes, for smooth problems with very many genes and few inequality constraints, from the program's gradient and, with `fitness.constraints`, the constraints' values and Jacobian. Each iteration is one evaluation. It starts from a random point and ends the run by itself when it has converged (the KKT residual within `kkt_tolerance`, or a step within `step_tolerance` of each gene's range), with the stop reason `"converged"`; a `[stop]` condition is still needed. A run that converges to a point infeasible by rounding ends with a restoration step onto the feasible side of its active constraints (`restoration = false` turns it off). `method = "gcmma"` is the globally convergent form: more evaluations, convergence from any start. `asymptote_initial` (above 0) is the asymptotes' first distance from the point and `move_limit` (above 0) the largest step, as fractions of each gene's range; `asymptote_decrease` (above 0 and at most 1) and `asymptote_increase` (at least 1) move them for oscillating and steady genes. `constraint_cost` (above 0) is the cost of the artificial variable that relaxes each constraint: above the constraints' multipliers at the solution. `parallel_sums = true` sums over the genes on several threads, with the same results. One worker is enough: it evaluates one genome at a time. - `first-order`: a first-order method, for smooth functions: each generation evaluates a point and its gradient, and steps along the gradient by a step rule, without a line search; the point stays in the bounds by projection. `step` is `{ type = "gradient", learning_rate }`, `{ type = "momentum", learning_rate, momentum }` (Polyak's heavy ball), `{ type = "nesterov", learning_rate, momentum }` (Nesterov's accelerated gradient, as Sutskever et al. state it), `{ type = "adam", learning_rate, beta1, beta2, epsilon }` (Kingma and Ba; all optional, 0.001, 0.9, 0.999 and 1e-8 by default) or `{ type = "adamw", learning_rate, beta1, beta2, epsilon, weight_decay }` (Loshchilov and Hutter's decoupled weight decay, `weight_decay` required), with the learning rate greater than 0 in the units of the genes, `momentum`, `beta1` and `beta2` from 0 to less than 1, `epsilon` greater than 0 and `weight_decay` 0 or more. `gradients` and `difference_step` are as for `lbfgsb`: the program's gradient with `fitness.gradient = true` (one evaluation per generation), or finite differences, asked in the same generation as the point (`"auto"` up to 10,000 genes whose bounds differ). The run ends by itself with the stop reason `"converged"` when the gradient's largest component is within `gradient_tolerance` (0 at a bound it points out of) or the last step moved no gene by more than `step_tolerance` relative to max(1, |x|), both 0 or more; a `[stop]` condition is still needed. `restarts = ` starts again from a random point that many times (at least 1) after converging. diff --git a/src/bin/genoxide/config.rs b/src/bin/genoxide/config.rs index 4cff1bd7..2c3f3452 100644 --- a/src/bin/genoxide/config.rs +++ b/src/bin/genoxide/config.rs @@ -228,6 +228,20 @@ pub enum Algorithm { /// Random restarts after convergence; none if unset. restarts: Option, }, + Bo { + seed: Option, + /// The points of the initial design; 2(n + 1) if unset. + initial_points: Option, + #[serde(default, deserialize_with = "bo_acquisition")] + acquisition: Option, + kernel: Option, + #[serde(default, deserialize_with = "bo_noise")] + noise: Option, + output: Option, + raw_samples: Option, + acquisition_starts: Option, + hyperparameter_starts: Option, + }, NelderMead { seed: Option, #[serde(default, deserialize_with = "nelder_mead_coefficients")] @@ -310,6 +324,8 @@ named! { de_control: Option = "control"; de_restarts: Option = "restarts"; nelder_mead_coefficients: Option = "coefficients"; + bo_acquisition: Option = "acquisition"; + bo_noise: Option = "noise"; first_order_step: Option = "step"; } @@ -440,6 +456,60 @@ pub struct OnStagnation { pub patience: u64, } +/// Bayesian optimization's acquisition function: `"log-ei"`, `"ei"`, `{ type = "pi", xi }` or +/// `{ type = "ucb", beta }`. +#[derive(Clone, Copy, Debug, Deserialize, Serialize)] +#[serde( + untagged, + expecting = "\"log-ei\", \"ei\", { type = \"pi\", xi } or { type = \"ucb\", beta }" +)] +pub enum BoAcquisition { + Named(BoAcquisitionName), + Table(BoAcquisitionTable), +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum BoAcquisitionName { + LogEi, + Ei, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize)] +#[serde(tag = "type", rename_all = "kebab-case", deny_unknown_fields)] +pub enum BoAcquisitionTable { + Pi { xi: f64 }, + Ucb { beta: f64 }, +} + +/// The kernel of Bayesian optimization's Gaussian process. +#[derive(Clone, Copy, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum Kernel { + Matern52, + SquaredExponential, +} + +/// The noise of Bayesian optimization's Gaussian process: a fixed variance, a fraction of the +/// values' variance, or `{ learned = }`. +#[derive(Clone, Copy, Debug, Deserialize, Serialize)] +#[serde( + untagged, + expecting = "a fixed variance (a number) or { learned = }" +)] +pub enum BoNoise { + Fixed(f64), + Learned { learned: f64 }, +} + +/// What Bayesian optimization's model fits: `"standardize"` or `"log"`. +#[derive(Clone, Copy, Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum BoOutput { + Standardize, + Log, +} + /// MMA's method: `"mma"` or `"gcmma"`. #[derive(Clone, Copy, Debug, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] diff --git a/src/bin/genoxide/run.rs b/src/bin/genoxide/run.rs index 216ec5a0..fa93c463 100644 --- a/src/bin/genoxide/run.rs +++ b/src/bin/genoxide/run.rs @@ -11,6 +11,7 @@ use genoxide::checkpoint; use genoxide::engine::asynchronous::MAX_WORKERS; use genoxide::genome::Representation; use genoxide::gradient::Gradients; +use genoxide::model::gp; use genoxide::multi::MultiObjectiveAlgorithm; use genoxide::observer::Report; use genoxide::operator::{Crossover, Mutate}; @@ -594,6 +595,69 @@ fn real_algorithm(real: Real, algorithm: config::Algorithm, context: &Context) - } generational(setting(builder.build())?, context) } + config::Algorithm::Bo { + seed, + initial_points, + acquisition, + kernel, + noise, + output, + raw_samples, + acquisition_starts, + hyperparameter_starts, + } => { + let mut builder = Bo::builder(real).objective(context.single_objective()?); + if let Some(seed) = seed { + builder = builder.seed(seed); + } + if let Some(points) = initial_points { + builder = builder.initial_points(points); + } + if let Some(acquisition) = acquisition { + builder = builder.acquisition(match acquisition { + config::BoAcquisition::Named(config::BoAcquisitionName::LogEi) => { + bo::Acquisition::LogExpectedImprovement + } + config::BoAcquisition::Named(config::BoAcquisitionName::Ei) => { + bo::Acquisition::ExpectedImprovement + } + config::BoAcquisition::Table(config::BoAcquisitionTable::Pi { xi }) => { + bo::Acquisition::ProbabilityOfImprovement { xi } + } + config::BoAcquisition::Table(config::BoAcquisitionTable::Ucb { beta }) => { + bo::Acquisition::UpperConfidenceBound { beta } + } + }); + } + if let Some(kernel) = kernel { + builder = builder.kernel(match kernel { + config::Kernel::Matern52 => gp::Kernel::Matern52, + config::Kernel::SquaredExponential => gp::Kernel::SquaredExponential, + }); + } + if let Some(noise) = noise { + builder = builder.noise(match noise { + config::BoNoise::Fixed(variance) => gp::Noise::Fixed(variance), + config::BoNoise::Learned { learned } => gp::Noise::Learned { min: learned }, + }); + } + if let Some(output) = output { + builder = builder.output(match output { + config::BoOutput::Standardize => bo::Output::Standardize, + config::BoOutput::Log => bo::Output::Log, + }); + } + if let Some(samples) = raw_samples { + builder = builder.raw_samples(samples); + } + if let Some(starts) = acquisition_starts { + builder = builder.acquisition_starts(starts); + } + if let Some(starts) = hyperparameter_starts { + builder = builder.hyperparameter_starts(starts); + } + generational(setting(builder.build())?, context) + } config::Algorithm::NelderMead { seed, coefficients, @@ -777,6 +841,7 @@ where config::Algorithm::NelderMead { .. } => { Err("`nelder-mead` needs a real genome".to_string()) } + config::Algorithm::Bo { .. } => Err("`bo` needs a real genome".to_string()), config::Algorithm::Lbfgsb { .. } => Err("`lbfgsb` needs a real genome".to_string()), config::Algorithm::Mma { .. } => Err("`mma` needs a real genome".to_string()), config::Algorithm::FirstOrder { .. } => { diff --git a/tests/cli.rs b/tests/cli.rs index b17a1060..a5ca6cd2 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -764,6 +764,120 @@ fn stop_and_report_mistakes_are_found_by_check() { std::fs::remove_dir_all(&directory).unwrap(); } +const BO: &str = r#" +report = "off" +[genome] +type = "real" +length = 2 +bounds = [-5.0, 5.0] +[fitness] +builtin = "sphere" +objectives = ["minimize"] +workers = 2 +[algorithm] +type = "bo" +seed = 1 +[stop] +evaluations = 25 +"#; + +// a run file of `BO` with these settings +fn bo(settings: &str) -> String { + BO.replace( + "type = \"bo\"", + &format!( + "type = \"bo\" +{settings}" + ), + ) +} + +#[test] +fn bayesian_optimization_runs() { + let directory = directory("bo"); + let run = |text: &str| untimed(run(&directory, "run.toml", text, &[]).unwrap()); + let default = run(BO); + assert_eq!(default["stop_reason"], "evaluations"); + assert_eq!(default["evaluations"], 25); + assert!(default["fitness"].as_f64().unwrap() < 1e-6, "{default}"); + // the settings' defaults + let explicit = run(&bo("initial_points = 6 +acquisition = \"log-ei\" +kernel = \"matern52\" + noise = 0.0 +output = \"standardize\" +raw_samples = 1000 +acquisition_starts = 10 + hyperparameter_starts = 5")); + assert_eq!(explicit, default); + // the other settings + for settings in [ + "acquisition = \"ei\"", + "acquisition = { type = \"pi\", xi = 0.01 }", + "acquisition = { type = \"ucb\", beta = 4.0 }", + "kernel = \"squared-exponential\"", + "noise = { learned = 1e-6 }", + "output = \"log\"", + ] { + let other = run(&bo(settings)); + assert_eq!(other["evaluations"], 25, "{settings}"); + assert!( + other["fitness"].as_f64().unwrap() < 0.1, + "{settings}: {other}" + ); + assert_ne!(other, default, "{settings}"); + } + // invalid settings + for (settings, message) in [ + ("acquisition = \"ucb\"", "acquisition"), + ("acquisition = { type = \"ucb\", beta = -1.0 }", "beta"), + ("noise = { learned = 0.0 }", "noise"), + ("initial_points = 0", "initial_points"), + ("raw_samples = 0", "raw_samples"), + ] { + let error = run_error(&directory, &bo(settings)); + assert!(error.contains(message), "{settings}: {error}"); + } + let error = run_error( + &directory, + &BO.replace( + "type = \"real\" +length = 2 +bounds = [-5.0, 5.0]", + "type = \"binary\" +length = 4", + ) + .replace("builtin = \"sphere\"", "builtin = \"one-max\""), + ); + assert!(error.contains("`bo` needs a real genome"), "{error}"); + std::fs::remove_dir_all(&directory).unwrap(); +} + +// the error of a run file that doesn't run +fn run_error(directory: &Path, text: &str) -> String { + run(directory, "error.toml", text, &[]).unwrap_err() +} + +#[test] +fn bayesian_optimization_resumes_from_checkpoints() { + let directory = directory("bo-resume"); + let text = |evaluations: u64| { + format!( + "{}[checkpoint] +path = \"run.ckpt\" +every = 3 +", + BO.replace("evaluations = 25", &format!("evaluations = {evaluations}")) + ) + }; + let whole = run(&directory, "whole.toml", &text(20), &[]).unwrap(); + let first = run(&directory, "part.toml", &text(12), &[]).unwrap(); + assert_eq!(first["evaluations"], 12); + let resumed = run(&directory, "part.toml", &text(20), &["--resume"]).unwrap(); + assert_eq!(untimed(resumed), untimed(whole)); + std::fs::remove_dir_all(&directory).unwrap(); +} + const NELDER_MEAD: &str = r#" report = "off" [genome] From 1889a2584a6158b9696141c1ac031aebe820f147 Mon Sep 17 00:00:00 2001 From: tachsin Date: Fri, 2 Oct 2026 12:47:11 +0300 Subject: [PATCH 11/13] docs: Bayesian optimization in the guides, and the sources read for it AGENTS.md gains a section on Bo with a program that runs in CI (the search, then a polish of the model's mean), rows in both method tables and in troubleshooting, and the panics of a point of the wrong length. The README, the crate docs, the Python README, both llms.txt, the features list and the packages' descriptions name Bayesian optimization; the roadmap marks the first part of batch B done and lists the second. The plan records what was verified in Rasmussen and Williams (2006), Jones, Schonlau and Welch (1998), Loeppky, Sacks and Welch (2009), Snoek, Larochelle and Adams (2012) and Srinivas et al. (2010), and where the implementation departs from its design notes, with the measurements: no noise by default, the log transform's offset, re-evaluation. --- AGENTS.md | 56 ++++++++++++++++++++++++++- Cargo.toml | 2 +- README.md | 3 +- ROADMAP.md | 3 +- docs/features.md | 3 ++ docs/optimization-plan.md | 62 +++++++++++++++++++++++++++++- docs/site/llms.txt | 2 +- python/README.md | 1 + python/pyproject.toml | 2 +- site/lib/projects/genoxide/llms.js | 2 +- src/lib.rs | 5 ++- 11 files changed, 129 insertions(+), 12 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e92aa438..4907ba9f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,6 +42,7 @@ fn main() -> genoxide::Result<()> { | A neural network's weights (neuroevolution) | `nn::Mlp::new([4, 8, 1], nn::Activation::Tanh)?.representation(-1.0..=1.0)?`, `nn::Elman` (recurrent) ([template](#neuroevolution-a-networks-weights-by-cma-es)) | `Reals` | none: `Cmaes` (up to a few hundred weights), `OpenEs` (thousands and more) | none | | A smooth function of many reals, its gradient noisy (mini-batches) or a step set by a learning-rate schedule (model fitting, up to millions of parameters) | `Real::uniform(n, lo..=hi)` ([template](#first-order-methods-adam-momentum-nesterov)) | `Reals` | none: `FirstOrder` | none | | Smooth, with gradients: very many reals (up to millions), few inequality constraints | `Real::uniform(n, lo..=hi)` ([template](#many-variables-few-constraints-mma)) | `Reals` | none: `Mma` | none | +| An expensive function of reals: tens to a few hundred evaluations, up to about 10 to 20 genes | `Real::new(...)` ([template](#bayesian-optimization-expensive-functions)) | `Reals` | none: `Bo` | none | | Continuous problem | Method | |---|---| @@ -49,6 +50,7 @@ fn main() -> genoxide::Result<()> { | Smooth or not, a few genes, no gradient | `NelderMead` | | Multimodal, rotated or badly conditioned, up to a few hundred genes | `Cmaes` (with `Restarts::Ipop`), `De`; then `Lbfgsb` from the best to polish it | | Smooth, gradients of the score and of each constraint, very many genes and few inequality constraints | `Mma` (`Method::Gcmma` to converge from any start) | +| Expensive: tens to a few hundred evaluations, up to about 10 to 20 genes | `Bo` (Bayesian optimization); `bo::Output::Log` for values over orders of magnitude | | Smooth, solved best in stages (a smoothing, sharpness or penalty changed step by step) | `Continuation` around `FirstOrder`, `Lbfgsb`, `Mma`, `NelderMead` or `Cmaes` ([template](#continuation-stages-of-one-problem-the-state-kept)) | | Constrained beyond the box, without gradients | `De` or `Ga` with `(score, violation)` (Deb's rules) | @@ -1098,6 +1100,53 @@ fn main() -> genoxide::Result<()> { } ``` +### Bayesian optimization: expensive functions + +`Bo` on `Real` genomes, for functions so expensive that tens to a few hundred evaluations must do (a simulation of minutes, an experiment). A Gaussian process (`model::gp`) models the function from every evaluation, and an acquisition function of its posterior picks the next point. Generation 0 evaluates an initial design: the `.initial_genomes(...)`, and a Latin hypercube for the rest of `.initial_points(n)`, 2(n + 1) for n searched genes by default (Loeppky et al. 2009's 10n is for an accurate model of the whole box, more than a minimum needs). Each later generation evaluates one point: the model is fitted (hyperparameters by maximum likelihood, genoxide's L-BFGS-B with the analytic gradient, from the last fit's and random starts), the acquisition evaluated at 1,000 random points and maximized by L-BFGS-B from the best 10 and the best point so far. A point is never asked twice. The model's fit costs O(N³) for N evaluations: up to a few hundred evaluations, in up to about 10 to 20 genes; for a cheap function, `Cmaes` or `De`. + +| Setting | Default | +|---|---| +| `.acquisition(bo::Acquisition::...)` | `LogExpectedImprovement` (Ament et al. 2023: finite with its gradient where EI underflows); `ExpectedImprovement`, `ProbabilityOfImprovement { xi }`, `UpperConfidenceBound { beta }` (`set_acquisition` in `.control` for a schedule) | +| `.kernel(model::gp::Kernel::...)` | `Matern52` (ν = 5/2, a length scale per gene); `SquaredExponential` | +| `.noise(model::gp::Noise::...)` | `Fixed(0.0)`: the model interpolates the values, as suits a deterministic function; `Learned { min }` (e.g. 1e-6) for a noisy one | +| `.output(bo::Output::...)` | `Standardize`; `Log` (`ln(v − v_best + δ)`, δ the first quartile of the distances above the best) for values over orders of magnitude (Goldstein-Price's 3 to 10⁶) | +| `.raw_samples(n)`, `.acquisition_starts(k)`, `.hyperparameter_starts(k)` | 1000, 10, 5 | + +The model fits the scores to minimize (negated when maximizing); an invalid fitness enters it at the worst value of the others; a violation is ignored by the search (use a penalty). `bo.model()` is the `GaussianProcess` that chose the last point (`predict(&x)`, `predict_with_gradient`, `hyperparameters()`), `bo.acquisition_at(&x)` its acquisition; `reevaluate()` asks every point again. The model can be fitted on its own: `GaussianProcess::builder(real).fit(&points, &values)?` (unstable for one release). Batch and asynchronous Bayesian optimization, constraints and integer genes come in a later release. + +```rust +use genoxide::model::gp::GaussianProcess; +use genoxide::prelude::*; +use genoxide::problems::{Branin, Problem}; + +fn main() -> genoxide::Result<()> { + // Branin's function: three global minima of 0.397887, in tens of evaluations + let bo = Bo::builder(Branin.representation()).minimize().seed(1).build()?; + let mut engine = Engine::new(bo, Branin).stop_when(Stop::evaluations(30)); + let outcome = engine.run()?; + assert!(outcome.best_fitness().score().unwrap() < 0.397887 + 1e-3); + + // polish: a model of every evaluation, its mean minimized from the best point, evaluated once + let evaluated = engine.algorithm().population(); + let points: Vec = evaluated.iter().map(|x| x.genome().clone()).collect(); + let values: Vec = evaluated.iter().map(|x| x.fitness().unwrap().score().unwrap()).collect(); + let model = GaussianProcess::builder(Branin.representation()).fit(&points, &values)?; + let mean = Differentiable(|x: &Reals, gradient: &mut [f64]| { + model.predict_with_gradient(x, gradient, &mut [0.0; 2]).mean() + }); + let lbfgsb = Lbfgsb::builder(Branin.representation()) + .initial_genome(outcome.best_genome().clone()) + .minimize() + .seed(1) + .build()?; + let polished = Engine::new(lbfgsb, mean).stop_when(Stop::evaluations(1_000)).run()?; + assert!(Branin.evaluate(polished.best_genome()) < 0.397887 + 1e-4); + Ok(()) +} +``` + +Python: `gx.Bo(real, initial_points=None, acquisition="log-ei" | "ei" | gx.ProbabilityOfImprovement(xi) | gx.UpperConfidenceBound(beta), kernel="matern52", noise=0.0 | gx.model.gp.Learned(min), output="standardize" | "log", raw_samples=1000, acquisition_starts=10, hyperparameter_starts=5, ...)`; `gx.RunningBo` in `control` (`acquisition`, `model`, `acquisition_at(points)`); `gx.model.gp.GaussianProcess.fit(real, points, values)` (`predict(points)`, `predict_with_gradient(x)`). The `genoxide` program: `type = "bo"`. See `examples/bayesian_optimization`. + ### Ask / tell: evaluating outside the engine For fitness computed elsewhere (another process, async code). @@ -1145,7 +1194,7 @@ command = ["python3", "fitness.py"] # or builtin = "rastrigin" objectives = ["minimize"] [algorithm] -type = "ga" # ga, steady-ga, de, cmaes, pso, local-search, nelder-mead, lbfgsb, first-order, mma, nsga2 +type = "ga" # ga, steady-ga, de, cmaes, pso, local-search, nelder-mead, lbfgsb, first-order, mma, bo, nsga2 population_size = 50 select = { type = "tournament", size = 3 } crossover = { type = "simulated-binary", eta = 15.0 } @@ -1195,6 +1244,9 @@ every = 50 | Real-valued GA stuck in a local minimum | `PolynomialMutation` with eta 20, or a larger `GaussianMutation` sigma | | `StopReason::Stalled`: 10 000 generations of copies only (e.g. `mutation_rate(0.0)`) | A mutation rate above 0, or add `Stop::generations(n)` or `Stop::stagnation(n)` | | Early stagnation (too little diversity) | A larger population, smaller tournament or higher mutation rate; or `Stop::stagnation` and restart | +| `Bo` stalls above the minimum of a function whose values span orders of magnitude | `.output(bo::Output::Log)` | +| `Bo` on a noisy function chases the noise | `.noise(model::gp::Noise::Learned { min: 1e-6 })` | +| `Bo` takes long per step after hundreds of evaluations | The model costs O(N³) for N evaluations: for a cheap function, `Cmaes` or `De` | | Slow with a cheap fitness function | `--release`; `.parallel(true)` only for expensive fitness; `.parallel_breeding(true)` (`Ga`, `De`, `Es`) when breeding takes much of a generation | ## Guarantees to rely on @@ -1202,4 +1254,4 @@ every = 50 - **Reproducible:** a seed gives the same results on every platform and thread count, parallel or not. The exception is a fitness function that calls the platform's `sin`, `cos`, `exp` and the like (`f64::sin`, numpy): their last bit can differ between operating systems, and long runs drift apart. `genoxide::math::{sin, cos, tan, exp, ln, powf, powi, atan2, ...}` are the same to the bit everywhere, at native speed; `problems` and `multi::problems` use them. - **Ties:** the earlier individual wins. - **The best is kept:** `outcome.best()` is the best individual ever evaluated. -- **Errors, not panics,** for invalid settings, including sizes above 2^24. The only panics (`# Panics`): an index out of bounds (`Bits::set`, `Order::swap`), a `problems` or `multi::problems` constructor with too few dimensions or variables (or a radius that isn't above 0, or none from the paper for `C1Dtlz3::new` and `ConvexC2Dtlz2::new`), a `Batch` returning no score for a single genome, a gradient or constraint-values slice of the wrong length for a test problem's `evaluate_with`, and an input, output or observation slice of the wrong length for an `nn` network or a `control` task. +- **Errors, not panics,** for invalid settings, including sizes above 2^24. The only panics (`# Panics`): an index out of bounds (`Bits::set`, `Order::swap`), a `problems` or `multi::problems` constructor with too few dimensions or variables (or a radius that isn't above 0, or none from the paper for `C1Dtlz3::new` and `ConvexC2Dtlz2::new`), a `Batch` returning no score for a single genome, a gradient or constraint-values slice of the wrong length for a test problem's `evaluate_with`, an input, output or observation slice of the wrong length for an `nn` network or a `control` task, and a point of the wrong length for a `model::gp::GaussianProcess`'s predictions or `Bo::acquisition_at`. diff --git a/Cargo.toml b/Cargo.toml index 8ab70912..09cf1b0d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ name = "genoxide" version.workspace = true edition = "2024" rust-version = "1.88" -description = "Optimization for Rust and Python: genetic algorithms, CMA-ES, differential evolution, multi-objective, genetic programming, L-BFGS-B, Adam, MMA and more, reproducible on every platform" +description = "Optimization for Rust and Python: genetic algorithms, CMA-ES, differential evolution, multi-objective, genetic programming, L-BFGS-B, Adam, MMA, Bayesian optimization and more, reproducible on every platform" license = "MIT OR Apache-2.0" repository = "https://github.com/tachsin/genoxide" readme = "README.md" diff --git a/README.md b/README.md index 9beb17b1..a8307861 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ [![License](https://img.shields.io/crates/l/genoxide.svg)](#license) [![Benchmarks](https://img.shields.io/badge/benchmarks-16_libraries-ce422b)](https://tachsin.gr/projects/genoxide/benchmarks) -**Optimization for Rust and Python: genetic algorithms, evolution strategies, CMA-ES, differential evolution, particle swarms, genetic programming, multi-objective optimization, local search and gradient-based methods (L-BFGS-B, Adam, MMA) in one library. A seed gives the same results, to the bit, on every platform and thread count.** +**Optimization for Rust and Python: genetic algorithms, evolution strategies, CMA-ES, differential evolution, particle swarms, genetic programming, multi-objective optimization, local search, gradient-based methods (L-BFGS-B, Adam, MMA) and Bayesian optimization in one library. A seed gives the same results, to the bit, on every platform and thread count.** ## Install @@ -48,6 +48,7 @@ fn main() -> genoxide::Result<()> { - **Evolution strategies, CMA-ES, differential evolution, particle swarms:** with IPOP and BIPOP restarts, JADE, SHADE and L-SHADE. - **Local search:** hill climbing, simulated annealing, tabu search, iterated local search, and the Nelder-Mead simplex method with random restarts. - **Gradient-based:** L-BFGS-B for smooth functions with bounds, from a few variables to millions, and gradient descent, momentum, Nesterov, Adam and AdamW with learning-rate schedules, with gradients supplied or by finite differences; MMA and GCMMA, the method of moving asymptotes, for millions of variables with few constraints, from supplied gradients and constraint Jacobians. Continuation runs any of them through stages of one problem, a smooth version first and sharper ones after, with the optimizer's state kept between stages. +- **Bayesian optimization:** for expensive functions, where tens to a few hundred evaluations must do: a Gaussian process with a Matérn or squared exponential kernel, its hyperparameters by maximum likelihood, and the log expected improvement, expected improvement, probability of improvement or confidence bound, maximized by L-BFGS-B with their gradients; a log transform for values that span orders of magnitude. The Gaussian process can be fitted and queried on its own. - **Genetic programming:** strongly typed trees of your own primitives, evolved into programs and formulas, with subtree and one-point crossover, subtree, point, hoist, shrink and constant mutation, bloat control, and fast evaluation over data. - **Neuroevolution:** multilayer perceptrons and recurrent networks whose weights evolve, NEAT, which evolves networks' structure too, and pole-balancing control tasks for them to solve. - **Multi-objective:** NSGA-II, NSGA-III, SPEA2, MOEA/D and SMS-EMOA, with quality indicators. diff --git a/ROADMAP.md b/ROADMAP.md index 87e0695e..d3704f88 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -227,7 +227,8 @@ The plan: [docs/gp-neuroevolution-plan.md](docs/gp-neuroevolution-plan.md). ### 0.13: Bayesian optimization - [x] EI, log-EI, UCB and PI; Latin hypercube designs; portable `erf`, `erfc` and `erfcx` (batch B, the parts that need neither linear algebra nor L-BFGS-B) -- [ ] Gaussian processes; batch, constrained and integer-variable Bayesian optimization, also on the asynchronous engine (batch B; after the linear algebra, [#373](https://github.com/tachsin/genoxide/issues/373), and L-BFGS-B) +- [x] Gaussian processes (`model::gp`, unstable for one release) and Bayesian optimization (`Bo`): an initial Latin hypercube of 2(n + 1) points, log-EI by default, an output transform, in Python (`gx.Bo`, `gx.model.gp`) and the `genoxide` program, with the `bayesian_optimization` example (batch B, its first part) +- [ ] Batch Bayesian optimization (Kriging believer, constant liar), `Incremental` for the asynchronous engine, constrained Bayesian optimization and integer genes, with the `bo_hartmann6`, `bo_asynchronous` and `bo_constrained` examples (batch B, its second part) ### 0.14: Constrained nonlinear programming - [ ] SQP and the augmented Lagrangian, on the constrained test problems (batch C) diff --git a/docs/features.md b/docs/features.md index c2e84935..0ff2e45e 100644 --- a/docs/features.md +++ b/docs/features.md @@ -67,6 +67,8 @@ What genoxide has on main; [docs.rs](https://docs.rs/genoxide) documents the lat - **First-order methods** (`FirstOrder`): steps along the gradient without a line search, for smooth problems with up to millions of variables, by a `first_order::Step` rule: gradient descent, Polyak's momentum, Nesterov's accelerated gradient (Sutskever et al.'s form), Adam (Kingma and Ba's Algorithm 1, with its bias correction) and AdamW (Loshchilov and Hutter's decoupled weight decay, Algorithm 2). One gradient per generation, supplied or by finite differences in the same round; points projected onto the bounds; convergence by the projected gradient or the step (`StopReason::Converged`); a learning rate and a schedule multiplier set by `control`, for schedules; an invalid point stepped back from; random restarts; re-evaluation that keeps the velocity and Adam's averages. O(n) memory and work per step, and no allocation after the first step (tested at a million genes); in Python (`gx.FirstOrder`, with `gradient=`) and the `genoxide` program (`type = "first-order"`, with the gradient protocol). - **MMA and GCMMA** (`Mma`): Svanberg's method of moving asymptotes and its globally convergent form, for smooth problems with very many variables (millions) and few inequality constraints, from the gradients of the score and of every constraint: convex separable approximations with moving asymptotes and the notes' default constants, solved through their dual in the constraints' multipliers (O(n·m) per iteration, no n × n matrix, nothing allocated after the first iteration, the sums in fixed chunks, optionally in parallel with the same bits); artificial variables that keep every subproblem feasible; GCMMA's inner iterations for convergence from any start; convergence by the KKT residual or the step (`StopReason::Converged`); a restoration step onto the feasible side of the active constraints at the end; multipliers, asymptotes and the KKT residual to read, re-evaluation and checkpoints. Reproduces the 2002 paper's table 8.1 and the 1987 paper's cantilever beam. - **Continuation** (`Continuation`, the `Continue` trait): a local method run through stages of one problem, a parameter of the fitness function changed between them by a closure (a smoothing, a p-norm's p, a penalty weight), each stage ending when the method has converged or after its budget of generations, and the run only after the last (`StopReason::Converged`). Each stage starts by evaluating the point again on the changed function, with the method's state kept (`Keep::State`: Adam's averages and step count, MMA's asymptotes, a Nelder-Mead simplex, a CMA-ES distribution; L-BFGS-B's pairs optionally) or only the point (`Keep::Point`). Each stage's generations, evaluations, best value and end reported by a callback and after the run; checkpoints that resume in their stage; nothing allocated per step (tested at a million genes); in Python (`gx.Continuation` around `FirstOrder`, `Lbfgsb` or `Mma`, with Python stage callbacks). +- **Bayesian optimization** (`Bo`): for expensive black-box functions, tens to a few hundred evaluations. An initial design (the initial genomes, then a Latin hypercube; 2(n + 1) points by default), then a point per generation: a Gaussian process fitted to every evaluation, and the acquisition function (log-EI by default, EI, PI through its logarithm, or the confidence bound, settable during a run) maximized from 1,000 raw samples by L-BFGS-B with its analytic gradient from the best 10 and the best point so far, on rayon with the winner by value then start. A point is never asked twice; invalid points enter the model at the worst value; an output transform (`bo::Output::Log`) for values over orders of magnitude. Reproducible to the bit on any platform and thread count, re-evaluation and checkpoints; the model that chose each point and its acquisition readable during a run; in Python (`gx.Bo`, `gx.RunningBo`) and the `genoxide` program (`type = "bo"`). +- **Gaussian processes** (`model::gp`, unstable for one release): regression with a constant mean, an ARD Matérn 5/2 or squared exponential kernel, no noise by default (the model interpolates a deterministic function) or learned noise, inputs scaled to the unit cube and outputs standardized; hyperparameters by maximum marginal likelihood (Rasmussen and Williams, eq. 2.30 and 5.9) with genoxide's L-BFGS-B from fixed and seeded random starts; the posterior mean and variance with their gradients; Cholesky factorizations with a growing jitter. In Python as `gx.model.gp`. - **Test problems** (`problems`): Sphere, the axis-parallel ellipsoid, Schwefel 1.2 and 2.26, Rastrigin, Rosenbrock, Ackley, Griewank, Levy, Zakharov, Styblinski-Tang, Michalewicz, Himmelblau, Branin, Goldstein-Price, the six-hump camel, Hartmann's functions in 3 and 6 dimensions, Shekel's with 5, 7 and 10 wells, Easom, the eggholder, Schaffer's F6, Schwefel 2.21 and 2.22, Dixon-Price, Trid, Powell's singular function, Beale, Booth, Matyas, Bohachevsky's three functions, the three-hump camel, Langermann, Shekel's foxholes and Kowalik, each with its bounds, its analytic gradient (all but the eggholder and Schwefel 2.21 and 2.22, which aren't differentiable), known optimum (or best known, for those found numerically) and reference, in Rust and Python. - **Constrained test problems:** CEC 2006's g01-g24 (`problems::cec2006`), and the engineering design problems (`problems::engineering`): the welded beam in two forms, the pressure vessel, the tension/compression spring, the speed reducer, the gear train (integer), the three-bar truss, the cantilever beam and the car side impact, each with its optimum or best known solution and references, in Rust and Python. Those with inequalities only give their constraints' values one by one to the algorithms that use them. @@ -127,6 +129,7 @@ cargo run --release --example polish # SHADE on Rastrigin, then L-B cargo run --release --example adam # 100,000 values smoothed to the exact answer, by Adam with a schedule cargo run --release --example mma # a million variables and one constraint, by MMA, to the closed-form minimum cargo run --release --example continuation # a tilted Rastrigin through 6 Gaussian smoothings, L-BFGS-B, to the global minimum +cargo run --release --example bayesian_optimization # Branin in 31 evaluations, a Gaussian process and log-EI, then a polish of its mean cargo run --release --example pressure_vessel # constrained mixed discrete-continuous design, SHADE cargo run --release --example welded_beam # constrained design in two forms, SHADE cargo run --release --example gear_train # integer genome, genetic algorithm diff --git a/docs/optimization-plan.md b/docs/optimization-plan.md index d637bef5..a522b0dd 100644 --- a/docs/optimization-plan.md +++ b/docs/optimization-plan.md @@ -17,7 +17,11 @@ the `genoxide` program, with its example; and continuation (`Continuation` and t trait of 2.12, for `FirstOrder`, `Mma`, `Lbfgsb`, `NelderMead` and `Cmaes`), in Python (around the three gradient methods) but not the `genoxide` program, whose configuration has no form for a stage's closure (a fitness program would have to be told its stage), with its example. Batch A3 is -done; the later batches aren't yet. genoxide has +done. Of batch B, the first part (B1): the Gaussian process (`model::gp`, public and documented as +unstable for one release), `Bo` with EI, log-EI, PI and UCB, the Latin hypercube design and the +output transform, in Python and the `genoxide` program, with the `bayesian_optimization` example; +batch BO, `Incremental` for `AsyncEngine`, constrained BO and integer genes (B2) aren't yet, nor +the later batches. genoxide has evolutionary and population-based methods (GA, ES, CMA-ES, DE, PSO, local search, NSGA-II and the other multi-objective algorithms). This plan adds the other families of a general optimization library: local derivative-free methods, gradient-based methods, constrained @@ -333,6 +337,60 @@ independent permutations), which the tests check directly. `erfcx` is genoxide's `erfc` times an exactly split `e^(x²)` below 26, the asymptotic series of Abramowitz and Stegun 7.1.23 above, checked against mpmath (`tests/reference/special_functions.py`) within 4 ulps. +**Batch B's first part (B1), read on 2026-10-02.** Rasmussen and Williams (2006), read in the +authors' PDF: **verified**, the formulas as implemented: the noisy observations (eq. 2.20), the +predictive mean and variance (2.25, 2.26) computed as Algorithm 2.1 does (L = chol(K + σ²I), +α = Lᵀ\(L\y), v = L\k*, V = k(x*, x*) − vᵀv, ln |K| from Σ ln Lᵢᵢ), the log marginal likelihood +(2.30, 5.8), a fixed mean function (2.37, 2.38), the squared exponential (4.9), the Matérn class's +ν = 5/2 member (4.17), the ARD distance M₂ = diag(ℓ)⁻² (5.1, 5.2) and the likelihood's gradient +(5.9), ½ tr((ααᵀ − K⁻¹) ∂K/∂θⱼ). The tests check two points worked out by hand against these +formulas to 1e-13, and the gradient against central differences. Jones, Schonlau and Welch +(1998), read: **verified**: the constant mean's generalized least squares estimate +1ᵀR⁻¹y / 1ᵀR⁻¹1 (eq. 5), which genoxide takes for the mean at every set of kernel hyperparameters +(the likelihood's maximum over the mean, so eq. 5.9 is the gradient of the likelihood so +maximized), the expected improvement (eq. 15) and its derivatives, −Φ in the prediction and φ in +its standard error (the bounds of their branch and bound), and the initial design of about 10k +points, which Loeppky, Sacks and Welch (2009), read, **verified**, study as a rule for prediction +accuracy (n = 10d), not optimization. Snoek, Larochelle and Adams (2012), read: **verified**, the ARD Matérn +5/2 kernel proposed for Bayesian optimization (section 3.1, eq. 5). Srinivas et al. (2010), read: +**verified**, the GP-UCB rule μ + β^½ σ (eq. 6). Log-EI's derivative is genoxide's own derivation +from Ament et al.'s log_h, log_h′ = Φ/h through the Mills ratio, with the asymptotic series +−z(1 + 2a − 6a² + 42a³), a = 1/z², below z = −64 (Abramowitz and Stegun 7.1.23, not re-read; the +coefficients and the next term's 414a⁴ checked with mpmath), where the Mills ratio's difference +loses ε z² of accuracy. Hvarfner et al.'s (2024) prior on the length scales: not implemented (an +option for higher dimensions, later). McKay et al. (1979) and Kushner (1964): not re-read. + +As implemented, and where it differs from the design notes above: **the noise** is none by +default (`Noise::Fixed(0.0)`, the jitter the only nugget), not learned from a floor of 1e-6: +genoxide's fitness functions are deterministic, and learned noise settled above its floor (an +absolute standard deviation of about 0.03 on Branin), smoothing over the differences near a +minimum. Over 20 seeds, to f* + 1e-4 within 80 evaluations, Branin was reached in 20 runs, the +six-hump camel in [−3, 3] × [−2, 2] in 20 and Hartmann 3 in 20, against 14, 17 and 12 with learned +noise; `Noise::Learned { min }` stays for noisy functions. **The hyperparameters**: the logarithms +of the length scales in [0.01, 100] of each range, σ_f² in [1e-3, 1e3] and a learned σ_n² from its +least to 1, all in the standardized units; L-BFGS-B (200 evaluations at most) from the last fit's +hyperparameters (or fixed values: length scales 0.5, σ_f² 1, σ_n² 1e-4) and 4 random points of the +box, each from a stream derived from the seed and the generation, the best likelihood winning, +the earlier start on ties. A narrower box for σ_f² and random starts drawn from a central part of +the box were measured, no better. **The acquisition's maximization** as planned: 1,000 raw samples, +L-BFGS-B from the best 10 and from the best point evaluated; the posterior variance floored at +1e-12 of the standardized variance there. **The output transform** (`bo::Output`), settled here: +`Standardize` by default, and `Log`, ln(v − v_best + δ) with δ the first quartile of the +distances above the best. Measured against other δ on Goldstein-Price, with learned noise, then +the default: the smallest positive distance made the best point an outlier (reached in 0 runs of +20), the median compressed too little near the best (18 of 20, a median of 52 evaluations), the +first quartile 20 of 20 (45). With the +noise-free default, to f* + 1e-3 within 80 evaluations: Goldstein-Price 19 runs of 20 (0 with +`Standardize`), the six-hump camel in [−5, 5]² 20 (4), Branin a median of 20 evaluations (30), but +Hartmann 3, whose values span less than an order of magnitude, 13 (20). **Invalid points** enter +the model at the worst valid value (an invalid fitness, or a score that isn't finite); with no +valid one, the next point is random. **Reevaluate** is implemented after all, contrary to 2.9: it +asks every evaluated point again, no random number drawn. `recommendation()` (noisy observations) +is left for later: with the noise-free default, the best evaluated point is the model's too. The +#381 evidence's typical counts, as measured here (20 seeds, to f* + 1e-3): Branin a median of 30 +evaluations, the six-hump camel in [−3, 3] × [−2, 2] 50, Hartmann 3 26, all within 80; Hartmann 6 +reached in 6 runs of 10 within 100 (48), the others in its local minimum −3.2032. + ### 1.6 Surrogate-assisted evolution, multi-fidelity, hybrids Brief and later: each needs the GP (batch B) and a design settled on real use. @@ -988,7 +1046,7 @@ a batch isn't done until every example reaches its optimum on the three platform | A1 | `linalg` (Cholesky, triangular solves, QR, the eigendecomposition moved from CMA-ES) with the dependency check of 2.8 (done, in-crate, #373: products, Cholesky with jitter, triangular solves and the eigendecomposition; QR, LDLᵀ and Bunch-Kaufman come with their first users); `Algorithm::is_finished` and `StopReason::Converged`; `Restarts` for local methods; Nelder-Mead (Gao-Han, 1965 option, speculative asks) | | `nelder_mead`, `nelder_mead_himmelblau` | | A2 | The extras of 2.3 (`Provided`, `Wanted`, `Extras`, `Evaluations`, `prepare`, `tell_evaluations`) in `Engine`; `Differentiable`, `gradient::Gradients`, finite differences, `gradient::check`; analytic gradients for the smooth problems; Moré-Thuente; L-BFGS-B | A1 | `lbfgsb`, `lbfgsb_bounds`, `polish` | | A3 | Momentum, Nesterov, Adam and AdamW (moved from D1); MMA and GCMMA, with supplied constraint values and Jacobians in `Extras` (the supplied half of batch C's constraint Jacobians; finite differences of constraints stay in C); `Continuation` and the `Continue` trait (2.12); the scale requirements of 2.13 for A2's and A3's methods, with the allocation test and the benchmarks | A2 | `adam`, `mma`, `continuation` | -| B | `model::gp` (kernels, hyperparameters by L-BFGS-B), portable `erf`/`erfc`/`erfcx` in `math`; `Bo` with EI, log-EI, UCB, PI; Latin hypercube; batch BO (Kriging believer, constant liar); `Incremental` for `AsyncEngine`; `Constrained` values (`constraint::Constraints`) and constrained BO; integer genes | A2 | `bayesian_optimization`, `bo_hartmann6`, `bo_asynchronous`, `bo_constrained` | +| B | B1 (done): `model::gp` (kernels, hyperparameters by L-BFGS-B), portable `erf`/`erfc`/`erfcx` in `math`; `Bo` with EI, log-EI, UCB, PI; Latin hypercube; the output transform. B2: batch BO (Kriging believer, constant liar); `Incremental` for `AsyncEngine`; `Constrained` values (`constraint::Constraints`) and constrained BO; integer genes | A2 | B1: `bayesian_optimization`; B2: `bo_hartmann6`, `bo_asynchronous`, `bo_constrained` | | C | Constraint Jacobians in `Extras` (supplied or by finite differences); the dense QP (Goldfarb-Idnani); SQP; the augmented Lagrangian (L-BFGS-B inner); `provides()` for CEC 2006 and the engineering problems; the Hock-Schittkowski selection | A2, B's `Constrained` | `sqp`, `sqp_welded_beam`, `augmented_lagrangian` | | D1 | BFGS, nonlinear CG with Hager-Zhang, trust-region Newton (Steihaug-CG and exact), Levenberg-Marquardt with `LeastSquares`; optional `dual` feature; MGH test set | A2 | `conjugate_gradient`, `trust_region`, `levenberg_marquardt`, `dual_numbers` | | D2 | BOBYQA, COBYLA, compass search / GPS, MADS with the progressive barrier, Powell's method (optional); basin hopping | A1, C's constraint values | `bobyqa`, `cobyla`, `mads` | diff --git a/docs/site/llms.txt b/docs/site/llms.txt index 8c2a6818..a3de1d2f 100644 --- a/docs/site/llms.txt +++ b/docs/site/llms.txt @@ -1,6 +1,6 @@ # genoxide -> Optimization for Rust and Python, in one library: genetic algorithms, evolution strategies, CMA-ES, differential evolution, particle swarms, local search, Nelder-Mead, genetic programming, neuroevolution, multi-objective optimization (NSGA-II, NSGA-III, MOEA/D, SPEA2, SMS-EMOA) and gradient-based methods (L-BFGS-B, Adam and momentum, MMA and GCMMA, and continuation in stages). A seed gives the same results, to the bit, on every platform and thread count. Invalid settings are errors, not panics. Each method is checked against its source paper. +> Optimization for Rust and Python, in one library: genetic algorithms, evolution strategies, CMA-ES, differential evolution, particle swarms, local search, Nelder-Mead, genetic programming, neuroevolution, multi-objective optimization (NSGA-II, NSGA-III, MOEA/D, SPEA2, SMS-EMOA) gradient-based methods (L-BFGS-B, Adam and momentum, MMA and GCMMA, and continuation in stages) and Bayesian optimization with Gaussian processes, for expensive functions. A seed gives the same results, to the bit, on every platform and thread count. Invalid settings are errors, not panics. Each method is checked against its source paper. Install: `cargo add genoxide` (Rust), `pip install genoxide` (Python), or `cargo install genoxide --features cli` for a program that runs an optimization described in TOML with any program as the fitness function. genoxide is pre-1.0: check the version and use the guide below, which matches the latest release. diff --git a/python/README.md b/python/README.md index 90c6856b..5c94c59a 100644 --- a/python/README.md +++ b/python/README.md @@ -10,6 +10,7 @@ The algorithms of [genoxide](https://github.com/tachsin/genoxide), a Rust librar - L-BFGS-B for smooth functions with a gradient, yours, finite differences or the test problems' own - first-order gradient methods for up to millions of parameters: gradient descent, momentum, Nesterov, Adam and AdamW - MMA and GCMMA, the method of moving asymptotes, for millions of variables with few constraints, from gradients +- Bayesian optimization for expensive functions, with Gaussian processes (`gx.model.gp`) you can also fit on their own - continuation: a gradient method through stages of one problem, its state kept between them - differential evolution, evolution strategies, CMA-ES and particle swarm optimization - NEAT, OpenAI's evolution strategy, neural networks and pole-balancing tasks, for neuroevolution diff --git a/python/pyproject.toml b/python/pyproject.toml index 235af82b..0e8d7ff9 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "genoxide" -description = "Optimization in Rust, for Python: genetic algorithms, CMA-ES, differential evolution, NSGA-II, L-BFGS-B, Adam, MMA and more, reproducible on every platform" +description = "Optimization in Rust, for Python: genetic algorithms, CMA-ES, differential evolution, NSGA-II, L-BFGS-B, Adam, MMA, Bayesian optimization and more, reproducible on every platform" readme = "README.md" license = "MIT OR Apache-2.0" license-files = ["licenses/LICENSE-MIT", "licenses/LICENSE-APACHE"] diff --git a/site/lib/projects/genoxide/llms.js b/site/lib/projects/genoxide/llms.js index ef15033b..b071ac74 100644 --- a/site/lib/projects/genoxide/llms.js +++ b/site/lib/projects/genoxide/llms.js @@ -4,7 +4,7 @@ */ export const LLMS_TXT = `# genoxide -> Optimization for Rust and Python, in one library: genetic algorithms, evolution strategies, CMA-ES, differential evolution, particle swarms, local search, Nelder-Mead, genetic programming, neuroevolution, multi-objective optimization (NSGA-II, NSGA-III, MOEA/D, SPEA2, SMS-EMOA) and gradient-based methods (L-BFGS-B, Adam and momentum, MMA and GCMMA, and continuation in stages). A seed gives the same results, to the bit, on every platform and thread count. Invalid settings are errors, not panics. Each method is checked against its source paper. +> Optimization for Rust and Python, in one library: genetic algorithms, evolution strategies, CMA-ES, differential evolution, particle swarms, local search, Nelder-Mead, genetic programming, neuroevolution, multi-objective optimization (NSGA-II, NSGA-III, MOEA/D, SPEA2, SMS-EMOA) gradient-based methods (L-BFGS-B, Adam and momentum, MMA and GCMMA, and continuation in stages) and Bayesian optimization with Gaussian processes, for expensive functions. A seed gives the same results, to the bit, on every platform and thread count. Invalid settings are errors, not panics. Each method is checked against its source paper. Install: \`cargo add genoxide\` (Rust), \`pip install genoxide\` (Python), or \`cargo install genoxide --features cli\` for a program that runs an optimization described in TOML with any program as the fitness function. genoxide is pre-1.0: check the version and use the guide below, which matches the latest release. diff --git a/src/lib.rs b/src/lib.rs index fd7edbac..b25961ba 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,7 @@ //! # genoxide //! -//! Optimization for Rust (and Python): evolutionary, local, gradient-based and multi-objective -//! methods in one library. A seed gives the same results, to the bit, on every platform and +//! Optimization for Rust (and Python): evolutionary, local, gradient-based, Bayesian and +//! multi-objective methods in one library. A seed gives the same results, to the bit, on every platform and //! thread count, parallel or not. //! //! | Problem | Method | @@ -10,6 +10,7 @@ //! | Real numbers in a box, no gradient | [`Cmaes`](algorithm::Cmaes), [`De`](algorithm::De), [`Pso`](algorithm::Pso), [`Es`](algorithm::Es), [`NelderMead`](algorithm::NelderMead) | //! | Smooth functions with a gradient | [`Lbfgsb`](algorithm::Lbfgsb), and [`FirstOrder`](algorithm::FirstOrder) (Adam, momentum) for millions of variables | //! | Many variables, few inequality constraints, with gradients | [`Mma`](algorithm::Mma) (MMA and GCMMA) | +//! | An expensive function: tens to a few hundred evaluations | [`Bo`](algorithm::Bo), Bayesian optimization, with the Gaussian processes of [`model::gp`] | //! | A smooth problem solved in stages (a smoothing, sharpness or penalty changed step by step) | [`Continuation`](algorithm::Continuation) around a local method, its state kept | //! | Several objectives at once | [`Nsga2`](multi::Nsga2), [`Nsga3`](multi::Nsga3), [`Moead`](multi::Moead), [`SmsEmoa`](multi::SmsEmoa) | //! | Programs and formulas | [`gp`]: tree genetic programming | From 27852c1a41e3d359928a2b303399af700c8a6b70 Mon Sep 17 00:00:00 2001 From: tachsin Date: Fri, 2 Oct 2026 12:54:35 +0300 Subject: [PATCH 12/13] fix(model): an error, not a model of NaNs, when the kernel matrix can't factor The final factorization of a fit can only fail if rounding defeats a jitter of the diagonal's scale; the fit then returns Error::InvalidSetting and Bo asks a random point instead. The module docs' link to Lbfgsb loses its redundant target, which the doc build rejected. --- examples/bayesian_optimization/README.md | 2 +- src/algorithm/bo.rs | 8 ++++++-- src/model/gp.rs | 26 +++++++++++++----------- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/examples/bayesian_optimization/README.md b/examples/bayesian_optimization/README.md index 3f655ed2..b2fb7e60 100644 --- a/examples/bayesian_optimization/README.md +++ b/examples/bayesian_optimization/README.md @@ -77,7 +77,7 @@ improvement that chose the next point, and a curve of the best value's distance ## Good results A good result is within 1e-4 of a global minimum, 0.397887, in tens of evaluations. The first -point the model chose, the 7th evaluation, is already 2.5 above it; the 16th is 0.043 above it, +point the model chose, the 7th evaluation, is 2.5 above it; the 16th is 0.043 above it, near (π, 2.275), and from there the search visits all three basins, the 24th within 1.3e-3 of (−π, 12.275), the 29th within 9.6e-5 of (3π, 2.475). The model of the 30 evaluations has its lowest mean at (9.422974, 2.475867), 0.397917, and the function there is 0.397909, 2.1e-5 above diff --git a/src/algorithm/bo.rs b/src/algorithm/bo.rs index e19b0579..47e80746 100644 --- a/src/algorithm/bo.rs +++ b/src/algorithm/bo.rs @@ -385,8 +385,12 @@ impl Bo { .derive(generation) .next_u64(), }; - let model = + let fitted = GaussianProcess::fit_unit(settings, scaling.clone(), x, &targets, self.warm.as_deref()); + // a kernel matrix that doesn't factor, even with jitter: a random point instead + let Ok(model) = fitted else { + return self.random_point(generation); + }; self.warm = Some(model.log_parameters().to_vec()); // the best point evaluated, by the model's values: the incumbent, and a start let mut incumbent: Option = None; @@ -953,7 +957,7 @@ mod tests { starts: 3, seed: 1, }; - let model = GaussianProcess::fit_unit(settings, scaling, x, &values, None); + let model = GaussianProcess::fit_unit(settings, scaling, x, &values, None).unwrap(); let (mean, scale) = model.standardization(); let best = values.iter().fold(f64::INFINITY, |a, &b| a.min(b)); (model, (best - mean) / scale) diff --git a/src/model/gp.rs b/src/model/gp.rs index e67b4099..a69e1397 100644 --- a/src/model/gp.rs +++ b/src/model/gp.rs @@ -56,7 +56,7 @@ //! [`fit`](GaussianProcessBuilder::fit) maximizes the log marginal likelihood (eq. 2.30 and 5.8) //! `ln p(y | X, θ) = −½ (y − m)ᵀ K_y⁻¹ (y − m) − ½ ln |K_y| − (n/2) ln 2π`, `K_y = σ_f² K + σ_n² I`, //! over the logarithms of the length scales, of `σ_f²` and of `σ_n²`, with genoxide's -//! [`Lbfgsb`](crate::algorithm::Lbfgsb) and the analytic gradient of eq. 5.9, +//! [`Lbfgsb`] and the analytic gradient of eq. 5.9, //! `∂/∂θⱼ ln p = ½ tr((ααᵀ − K_y⁻¹) ∂K_y/∂θⱼ)`, `α = K_y⁻¹(y − m)`. The search runs from //! [several starts](GaussianProcessBuilder::starts): the first from fixed values (length scales of //! 0.5 of each range, `σ_f²` the values' variance, a learned `σ_n²` 1e-4 of it or its least), the @@ -556,7 +556,7 @@ impl GaussianProcess { x: Vec, values: &[f64], warm: Option<&[f64]>, - ) -> GaussianProcess { + ) -> Result { let count = values.len(); let (y_mean, y_scale, y) = standardize(values); let dims = scaling.dims(); @@ -574,14 +574,18 @@ impl GaussianProcess { let log = best.unwrap_or(first); let (length_scales, signal, noise) = unpack(&log, dims, settings.noise); let mut workspace = Workspace::new(count); - // the box's parameters always factor with jitter up to the diagonal's scale, a - // positive definite K_y + (σ_f² + σ_n²) I - let jitter = data - .factor(&length_scales, signal, noise, &mut workspace) - .unwrap_or(f64::NAN); + // the box's parameters factor with jitter up to the diagonal's scale, a positive definite + // K_y + (σ_f² + σ_n²) I, unless rounding breaks even that + let Some(jitter) = data.factor(&length_scales, signal, noise, &mut workspace) else { + return Err(Error::InvalidSetting { + setting: "values", + reason: "the kernel matrix of the points doesn't factor, even with a jitter of its diagonal's scale" + .to_string(), + }); + }; let (mean, log_likelihood) = data.solve(&mut workspace, None); let Workspace { l, b, .. } = workspace; - GaussianProcess { + Ok(GaussianProcess { kernel: settings.kernel, scaling, x, @@ -597,7 +601,7 @@ impl GaussianProcess { factor: l, alpha: b, log_likelihood, - } + }) } } @@ -1022,9 +1026,7 @@ impl GaussianProcessBuilder { seed: self.seed, }; match &self.hyperparameters { - None => Ok(GaussianProcess::fit_unit( - settings, scaling, x, values, None, - )), + None => GaussianProcess::fit_unit(settings, scaling, x, values, None), Some(h) => self.with_hyperparameters(h, settings, scaling, x, values), } } From eaccfb590c53b0a1d04101fea2180837834ce839 Mon Sep 17 00:00:00 2001 From: tachsin Date: Fri, 2 Oct 2026 13:11:17 +0300 Subject: [PATCH 13/13] test(bo): the parallel run only with the parallel feature --- tests/bo.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/bo.rs b/tests/bo.rs index 5d808fc1..234ac180 100644 --- a/tests/bo.rs +++ b/tests/bo.rs @@ -121,9 +121,13 @@ fn every_acquisition_and_kernel_finds_branin_minima() { // the evaluated genomes of a run, in order, and its best fn run(bo: Bo, parallel: bool, evaluations: u64) -> (Vec>, Individual) { - let mut engine = Engine::new(bo, Branin) - .parallel(parallel) - .stop_when(Stop::evaluations(evaluations)); + let engine = Engine::new(bo, Branin).stop_when(Stop::evaluations(evaluations)); + // without the `parallel` feature, both runs are sequential + #[cfg(feature = "parallel")] + let engine = engine.parallel(parallel); + #[cfg(not(feature = "parallel"))] + let _ = parallel; + let mut engine = engine; let outcome = engine.run().unwrap(); let genomes = engine .algorithm()