diff --git a/HISTORY.md b/HISTORY.md index 314e46b2a..b9e493dad 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -13,6 +13,7 @@ - Add qd score to lunar lander example ({pr}`458`) - Raise error if `result_archive` and `archive` have different fields ({pr}`461`) +- Warn user if resampling for bounds takes too long in ESs ({pr}`462`) #### Documentation diff --git a/ribs/emitters/opt/_cma_es.py b/ribs/emitters/opt/_cma_es.py index 374738ec9..d0dc7076f 100644 --- a/ribs/emitters/opt/_cma_es.py +++ b/ribs/emitters/opt/_cma_es.py @@ -3,12 +3,15 @@ Adapted from Nikolaus Hansen's pycma: https://github.com/CMA-ES/pycma/blob/master/cma/purecma.py """ +import warnings + import numba as nb import numpy as np from threadpoolctl import threadpool_limits from ribs._utils import readonly -from ribs.emitters.opt._evolution_strategy_base import EvolutionStrategyBase +from ribs.emitters.opt._evolution_strategy_base import ( + BOUNDS_SAMPLING_THRESHOLD, BOUNDS_WARNING, EvolutionStrategyBase) class DecompMatrix: @@ -193,6 +196,7 @@ def ask(self, batch_size=None): # Resampling method for bound constraints -> sample new solutions until # all solutions are within bounds. remaining_indices = np.arange(batch_size) + sampling_itrs = 0 while len(remaining_indices) > 0: unscaled_params = self._rng.normal( 0.0, @@ -209,6 +213,11 @@ def ask(self, batch_size=None): # out of bounds). remaining_indices = remaining_indices[np.any(out_of_bounds, axis=1)] + # Warn if we have resampled too many times. + sampling_itrs += 1 + if sampling_itrs > BOUNDS_SAMPLING_THRESHOLD: + warnings.warn(BOUNDS_WARNING) + return readonly(self._solutions) def _calc_strat_params(self, num_parents): diff --git a/ribs/emitters/opt/_evolution_strategy_base.py b/ribs/emitters/opt/_evolution_strategy_base.py index f6ebee3e2..5a237614c 100644 --- a/ribs/emitters/opt/_evolution_strategy_base.py +++ b/ribs/emitters/opt/_evolution_strategy_base.py @@ -3,6 +3,17 @@ import numpy as np +# Number of times solutions can be resampled before triggering a warning. +BOUNDS_SAMPLING_THRESHOLD = 100 + +# Warning for resampling solutions too many times. +BOUNDS_WARNING = ( + "During bounds handling, this ES resampled at least " + f"{BOUNDS_SAMPLING_THRESHOLD} times. This may indicate that your solution " + "space bounds are too tight. When bounds are passed in, the ES resamples " + "until all solutions are within the bounds -- if the bounds are too tight " + "or the distribution is too large, the ES will resample forever.") + class EvolutionStrategyBase(ABC): """Base class for evolution strategy optimizers for use with emitters. diff --git a/ribs/emitters/opt/_lm_ma_es.py b/ribs/emitters/opt/_lm_ma_es.py index 3bfcc72f0..c6b109f20 100644 --- a/ribs/emitters/opt/_lm_ma_es.py +++ b/ribs/emitters/opt/_lm_ma_es.py @@ -3,11 +3,14 @@ Adapted from Nikolaus Hansen's pycma: https://github.com/CMA-ES/pycma/blob/master/cma/purecma.py """ +import warnings + import numba as nb import numpy as np from ribs._utils import readonly -from ribs.emitters.opt._evolution_strategy_base import EvolutionStrategyBase +from ribs.emitters.opt._evolution_strategy_base import ( + BOUNDS_SAMPLING_THRESHOLD, BOUNDS_WARNING, EvolutionStrategyBase) class LMMAEvolutionStrategy(EvolutionStrategyBase): @@ -144,6 +147,7 @@ def ask(self, batch_size=None): # Resampling method for bound constraints -> sample new solutions until # all solutions are within bounds. remaining_indices = np.arange(batch_size) + sampling_itrs = 0 while len(remaining_indices) > 0: z = self._rng.standard_normal( (len(remaining_indices), self.solution_dim)) # (_, n) @@ -159,6 +163,11 @@ def ask(self, batch_size=None): # out of bounds). remaining_indices = remaining_indices[np.any(out_of_bounds, axis=1)] + # Warn if we have resampled too many times. + sampling_itrs += 1 + if sampling_itrs > BOUNDS_SAMPLING_THRESHOLD: + warnings.warn(BOUNDS_WARNING) + return readonly(self._solutions) @staticmethod diff --git a/ribs/emitters/opt/_openai_es.py b/ribs/emitters/opt/_openai_es.py index db8dcb47f..714861653 100644 --- a/ribs/emitters/opt/_openai_es.py +++ b/ribs/emitters/opt/_openai_es.py @@ -2,11 +2,14 @@ See here for more info: https://arxiv.org/abs/1703.03864 """ +import warnings + import numpy as np from ribs._utils import readonly from ribs.emitters.opt._adam_opt import AdamOpt -from ribs.emitters.opt._evolution_strategy_base import EvolutionStrategyBase +from ribs.emitters.opt._evolution_strategy_base import ( + BOUNDS_SAMPLING_THRESHOLD, BOUNDS_WARNING, EvolutionStrategyBase) class OpenAIEvolutionStrategy(EvolutionStrategyBase): @@ -58,6 +61,12 @@ def __init__( # pylint: disable = super-init-not-called self._rng = np.random.default_rng(seed) self._solutions = None + if mirror_sampling and not (np.all(lower_bounds == -np.inf) and + np.all(upper_bounds == np.inf)): + raise ValueError("Bounds are currently not supported when using " + "mirror_sampling in OpenAI-ES; see " + "OpenAIEvolutionStrategy.ask() for more info.") + self.mirror_sampling = mirror_sampling # Default batch size should be an even number for mirror sampling. @@ -105,14 +114,23 @@ def ask(self, batch_size=None): # Resampling method for bound constraints -> sample new solutions until # all solutions are within bounds. remaining_indices = np.arange(batch_size) + sampling_itrs = 0 while len(remaining_indices) > 0: if self.mirror_sampling: + # Note that we sample batch_size // 2 here rather than + # accounting for len(remaining_indices). This is because we + # assume we only run this loop once when mirror_sampling is + # True. It is unclear how to do bounds handling when mirror + # sampling is involved since the two entries need to be + # mirrored. For instance, should we throw out both solutions if + # one is out of bounds? noise_half = self._rng.standard_normal( (batch_size // 2, self.solution_dim), dtype=self.dtype) self.noise = np.concatenate((noise_half, -noise_half)) else: self.noise = self._rng.standard_normal( - (batch_size, self.solution_dim), dtype=self.dtype) + (len(remaining_indices), self.solution_dim), + dtype=self.dtype) new_solutions = (self.adam_opt.theta[None] + self.sigma0 * self.noise) @@ -128,6 +146,11 @@ def ask(self, batch_size=None): # out of bounds). remaining_indices = remaining_indices[np.any(out_of_bounds, axis=1)] + # Warn if we have resampled too many times. + sampling_itrs += 1 + if sampling_itrs > BOUNDS_SAMPLING_THRESHOLD: + warnings.warn(BOUNDS_WARNING) + return readonly(self._solutions) def tell(self, ranking_indices, ranking_values, num_parents): diff --git a/ribs/emitters/opt/_sep_cma_es.py b/ribs/emitters/opt/_sep_cma_es.py index 80033dfb4..51632f04b 100644 --- a/ribs/emitters/opt/_sep_cma_es.py +++ b/ribs/emitters/opt/_sep_cma_es.py @@ -3,11 +3,14 @@ Adapted from Nikolaus Hansen's pycma: https://github.com/CMA-ES/pycma/blob/master/cma/purecma.py """ +import warnings + import numba as nb import numpy as np from ribs._utils import readonly -from ribs.emitters.opt._evolution_strategy_base import EvolutionStrategyBase +from ribs.emitters.opt._evolution_strategy_base import ( + BOUNDS_SAMPLING_THRESHOLD, BOUNDS_WARNING, EvolutionStrategyBase) class DiagonalMatrix: @@ -148,6 +151,7 @@ def ask(self, batch_size=None): # Resampling method for bound constraints -> sample new solutions until # all solutions are within bounds. remaining_indices = np.arange(batch_size) + sampling_itrs = 0 while len(remaining_indices) > 0: unscaled_params = self._rng.normal( 0.0, @@ -164,6 +168,11 @@ def ask(self, batch_size=None): # out of bounds). remaining_indices = remaining_indices[np.any(out_of_bounds, axis=1)] + # Warn if we have resampled too many times. + sampling_itrs += 1 + if sampling_itrs > BOUNDS_SAMPLING_THRESHOLD: + warnings.warn(BOUNDS_WARNING) + return readonly(self._solutions) @staticmethod diff --git a/tests/emitters/evolution_strategy_emitter_test.py b/tests/emitters/evolution_strategy_emitter_test.py index c7b4de463..543675db0 100644 --- a/tests/emitters/evolution_strategy_emitter_test.py +++ b/tests/emitters/evolution_strategy_emitter_test.py @@ -95,3 +95,39 @@ def test_sphere(es): measures_batch = solution_batch[:, :2] add_info = archive.add(solution_batch, objective_batch, measures_batch) emitter.tell(solution_batch, objective_batch, measures_batch, add_info) + + +if __name__ == "__main__": + # For testing bounds handling. Run this file: + # python tests/emitters/evolution_strategy_emitter_test.py + # The below code should show the resampling warning indicating that the ES + # resampled too many times. This test cannot be included in pytest because + # it is designed to hang. Comment out the different emitters to test + # different ESs. + + archive = GridArchive(solution_dim=31, + dims=[20, 20], + ranges=[(-1.0, 1.0)] * 2) + emitter = EvolutionStrategyEmitter(archive, + x0=np.zeros(31), + sigma0=1.0, + bounds=[(0, 1.0)] * 31, + es="cma_es") + # emitter = EvolutionStrategyEmitter(archive, + # x0=np.zeros(31), + # sigma0=1.0, + # bounds=[(0, 1.0)] * 31, + # es="sep_cma_es") + # emitter = EvolutionStrategyEmitter(archive, + # x0=np.zeros(31), + # sigma0=1.0, + # bounds=[(0, 1.0)] * 31, + # es="lm_ma_es") + # emitter = EvolutionStrategyEmitter(archive, + # x0=np.zeros(31), + # sigma0=1.0, + # bounds=[(0, 1.0)] * 31, + # es="openai_es", + # es_kwargs={"mirror_sampling": False}) + + emitter.ask()