From 8d7394d7e19a12bce2542bc6d91789e3232f2068 Mon Sep 17 00:00:00 2001 From: Bryon Tjanaka <38124174+btjanaka@users.noreply.github.com> Date: Thu, 14 Mar 2024 02:30:56 -0700 Subject: [PATCH] Warn user if resampling for bounds takes too long in ESs (#462) ## Description A common error when using bounds is that CMA-ES or another ES can hang due to resampling, as solutions that fall outside of the bounds need to be resampled until they are within bounds. This PR adds a warning so that users will at least know that this behavior is occurring. We are still unclear how to deal with bounds properly, as it is also an open research question. #392 has proposed clipping the solutions after a set number of iterations of resampling but it is unclear if this is the best solution. ## TODO - [x] Fix slight issue with how OpenAI-ES handles resampling - [x] Add tests -> since this behavior is supposed to make the tests hang, we just put this as a script in one of the tests that can be manually run - [x] Modify ESs ## Status - [x] I have read the guidelines in [CONTRIBUTING.md](https://github.com/icaros-usc/pyribs/blob/master/CONTRIBUTING.md) - [x] I have formatted my code using `yapf` - [x] I have tested my code by running `pytest` - [x] I have linted my code with `pylint` - [x] I have added a one-line description of my change to the changelog in `HISTORY.md` - [x] This PR is ready to go --- HISTORY.md | 1 + ribs/emitters/opt/_cma_es.py | 11 +++++- ribs/emitters/opt/_evolution_strategy_base.py | 11 ++++++ ribs/emitters/opt/_lm_ma_es.py | 11 +++++- ribs/emitters/opt/_openai_es.py | 27 ++++++++++++-- ribs/emitters/opt/_sep_cma_es.py | 11 +++++- .../evolution_strategy_emitter_test.py | 36 +++++++++++++++++++ 7 files changed, 103 insertions(+), 5 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 314e46b2a..b9e493dad 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -13,6 +13,7 @@ - Add qd score to lunar lander example ({pr}`458`) - Raise error if `result_archive` and `archive` have different fields ({pr}`461`) +- Warn user if resampling for bounds takes too long in ESs ({pr}`462`) #### Documentation diff --git a/ribs/emitters/opt/_cma_es.py b/ribs/emitters/opt/_cma_es.py index 374738ec9..d0dc7076f 100644 --- a/ribs/emitters/opt/_cma_es.py +++ b/ribs/emitters/opt/_cma_es.py @@ -3,12 +3,15 @@ Adapted from Nikolaus Hansen's pycma: https://github.com/CMA-ES/pycma/blob/master/cma/purecma.py """ +import warnings + import numba as nb import numpy as np from threadpoolctl import threadpool_limits from ribs._utils import readonly -from ribs.emitters.opt._evolution_strategy_base import EvolutionStrategyBase +from ribs.emitters.opt._evolution_strategy_base import ( + BOUNDS_SAMPLING_THRESHOLD, BOUNDS_WARNING, EvolutionStrategyBase) class DecompMatrix: @@ -193,6 +196,7 @@ def ask(self, batch_size=None): # Resampling method for bound constraints -> sample new solutions until # all solutions are within bounds. remaining_indices = np.arange(batch_size) + sampling_itrs = 0 while len(remaining_indices) > 0: unscaled_params = self._rng.normal( 0.0, @@ -209,6 +213,11 @@ def ask(self, batch_size=None): # out of bounds). remaining_indices = remaining_indices[np.any(out_of_bounds, axis=1)] + # Warn if we have resampled too many times. + sampling_itrs += 1 + if sampling_itrs > BOUNDS_SAMPLING_THRESHOLD: + warnings.warn(BOUNDS_WARNING) + return readonly(self._solutions) def _calc_strat_params(self, num_parents): diff --git a/ribs/emitters/opt/_evolution_strategy_base.py b/ribs/emitters/opt/_evolution_strategy_base.py index f6ebee3e2..5a237614c 100644 --- a/ribs/emitters/opt/_evolution_strategy_base.py +++ b/ribs/emitters/opt/_evolution_strategy_base.py @@ -3,6 +3,17 @@ import numpy as np +# Number of times solutions can be resampled before triggering a warning. +BOUNDS_SAMPLING_THRESHOLD = 100 + +# Warning for resampling solutions too many times. +BOUNDS_WARNING = ( + "During bounds handling, this ES resampled at least " + f"{BOUNDS_SAMPLING_THRESHOLD} times. This may indicate that your solution " + "space bounds are too tight. When bounds are passed in, the ES resamples " + "until all solutions are within the bounds -- if the bounds are too tight " + "or the distribution is too large, the ES will resample forever.") + class EvolutionStrategyBase(ABC): """Base class for evolution strategy optimizers for use with emitters. diff --git a/ribs/emitters/opt/_lm_ma_es.py b/ribs/emitters/opt/_lm_ma_es.py index 3bfcc72f0..c6b109f20 100644 --- a/ribs/emitters/opt/_lm_ma_es.py +++ b/ribs/emitters/opt/_lm_ma_es.py @@ -3,11 +3,14 @@ Adapted from Nikolaus Hansen's pycma: https://github.com/CMA-ES/pycma/blob/master/cma/purecma.py """ +import warnings + import numba as nb import numpy as np from ribs._utils import readonly -from ribs.emitters.opt._evolution_strategy_base import EvolutionStrategyBase +from ribs.emitters.opt._evolution_strategy_base import ( + BOUNDS_SAMPLING_THRESHOLD, BOUNDS_WARNING, EvolutionStrategyBase) class LMMAEvolutionStrategy(EvolutionStrategyBase): @@ -144,6 +147,7 @@ def ask(self, batch_size=None): # Resampling method for bound constraints -> sample new solutions until # all solutions are within bounds. remaining_indices = np.arange(batch_size) + sampling_itrs = 0 while len(remaining_indices) > 0: z = self._rng.standard_normal( (len(remaining_indices), self.solution_dim)) # (_, n) @@ -159,6 +163,11 @@ def ask(self, batch_size=None): # out of bounds). remaining_indices = remaining_indices[np.any(out_of_bounds, axis=1)] + # Warn if we have resampled too many times. + sampling_itrs += 1 + if sampling_itrs > BOUNDS_SAMPLING_THRESHOLD: + warnings.warn(BOUNDS_WARNING) + return readonly(self._solutions) @staticmethod diff --git a/ribs/emitters/opt/_openai_es.py b/ribs/emitters/opt/_openai_es.py index db8dcb47f..714861653 100644 --- a/ribs/emitters/opt/_openai_es.py +++ b/ribs/emitters/opt/_openai_es.py @@ -2,11 +2,14 @@ See here for more info: https://arxiv.org/abs/1703.03864 """ +import warnings + import numpy as np from ribs._utils import readonly from ribs.emitters.opt._adam_opt import AdamOpt -from ribs.emitters.opt._evolution_strategy_base import EvolutionStrategyBase +from ribs.emitters.opt._evolution_strategy_base import ( + BOUNDS_SAMPLING_THRESHOLD, BOUNDS_WARNING, EvolutionStrategyBase) class OpenAIEvolutionStrategy(EvolutionStrategyBase): @@ -58,6 +61,12 @@ def __init__( # pylint: disable = super-init-not-called self._rng = np.random.default_rng(seed) self._solutions = None + if mirror_sampling and not (np.all(lower_bounds == -np.inf) and + np.all(upper_bounds == np.inf)): + raise ValueError("Bounds are currently not supported when using " + "mirror_sampling in OpenAI-ES; see " + "OpenAIEvolutionStrategy.ask() for more info.") + self.mirror_sampling = mirror_sampling # Default batch size should be an even number for mirror sampling. @@ -105,14 +114,23 @@ def ask(self, batch_size=None): # Resampling method for bound constraints -> sample new solutions until # all solutions are within bounds. remaining_indices = np.arange(batch_size) + sampling_itrs = 0 while len(remaining_indices) > 0: if self.mirror_sampling: + # Note that we sample batch_size // 2 here rather than + # accounting for len(remaining_indices). This is because we + # assume we only run this loop once when mirror_sampling is + # True. It is unclear how to do bounds handling when mirror + # sampling is involved since the two entries need to be + # mirrored. For instance, should we throw out both solutions if + # one is out of bounds? noise_half = self._rng.standard_normal( (batch_size // 2, self.solution_dim), dtype=self.dtype) self.noise = np.concatenate((noise_half, -noise_half)) else: self.noise = self._rng.standard_normal( - (batch_size, self.solution_dim), dtype=self.dtype) + (len(remaining_indices), self.solution_dim), + dtype=self.dtype) new_solutions = (self.adam_opt.theta[None] + self.sigma0 * self.noise) @@ -128,6 +146,11 @@ def ask(self, batch_size=None): # out of bounds). remaining_indices = remaining_indices[np.any(out_of_bounds, axis=1)] + # Warn if we have resampled too many times. + sampling_itrs += 1 + if sampling_itrs > BOUNDS_SAMPLING_THRESHOLD: + warnings.warn(BOUNDS_WARNING) + return readonly(self._solutions) def tell(self, ranking_indices, ranking_values, num_parents): diff --git a/ribs/emitters/opt/_sep_cma_es.py b/ribs/emitters/opt/_sep_cma_es.py index 80033dfb4..51632f04b 100644 --- a/ribs/emitters/opt/_sep_cma_es.py +++ b/ribs/emitters/opt/_sep_cma_es.py @@ -3,11 +3,14 @@ Adapted from Nikolaus Hansen's pycma: https://github.com/CMA-ES/pycma/blob/master/cma/purecma.py """ +import warnings + import numba as nb import numpy as np from ribs._utils import readonly -from ribs.emitters.opt._evolution_strategy_base import EvolutionStrategyBase +from ribs.emitters.opt._evolution_strategy_base import ( + BOUNDS_SAMPLING_THRESHOLD, BOUNDS_WARNING, EvolutionStrategyBase) class DiagonalMatrix: @@ -148,6 +151,7 @@ def ask(self, batch_size=None): # Resampling method for bound constraints -> sample new solutions until # all solutions are within bounds. remaining_indices = np.arange(batch_size) + sampling_itrs = 0 while len(remaining_indices) > 0: unscaled_params = self._rng.normal( 0.0, @@ -164,6 +168,11 @@ def ask(self, batch_size=None): # out of bounds). remaining_indices = remaining_indices[np.any(out_of_bounds, axis=1)] + # Warn if we have resampled too many times. + sampling_itrs += 1 + if sampling_itrs > BOUNDS_SAMPLING_THRESHOLD: + warnings.warn(BOUNDS_WARNING) + return readonly(self._solutions) @staticmethod diff --git a/tests/emitters/evolution_strategy_emitter_test.py b/tests/emitters/evolution_strategy_emitter_test.py index c7b4de463..543675db0 100644 --- a/tests/emitters/evolution_strategy_emitter_test.py +++ b/tests/emitters/evolution_strategy_emitter_test.py @@ -95,3 +95,39 @@ def test_sphere(es): measures_batch = solution_batch[:, :2] add_info = archive.add(solution_batch, objective_batch, measures_batch) emitter.tell(solution_batch, objective_batch, measures_batch, add_info) + + +if __name__ == "__main__": + # For testing bounds handling. Run this file: + # python tests/emitters/evolution_strategy_emitter_test.py + # The below code should show the resampling warning indicating that the ES + # resampled too many times. This test cannot be included in pytest because + # it is designed to hang. Comment out the different emitters to test + # different ESs. + + archive = GridArchive(solution_dim=31, + dims=[20, 20], + ranges=[(-1.0, 1.0)] * 2) + emitter = EvolutionStrategyEmitter(archive, + x0=np.zeros(31), + sigma0=1.0, + bounds=[(0, 1.0)] * 31, + es="cma_es") + # emitter = EvolutionStrategyEmitter(archive, + # x0=np.zeros(31), + # sigma0=1.0, + # bounds=[(0, 1.0)] * 31, + # es="sep_cma_es") + # emitter = EvolutionStrategyEmitter(archive, + # x0=np.zeros(31), + # sigma0=1.0, + # bounds=[(0, 1.0)] * 31, + # es="lm_ma_es") + # emitter = EvolutionStrategyEmitter(archive, + # x0=np.zeros(31), + # sigma0=1.0, + # bounds=[(0, 1.0)] * 31, + # es="openai_es", + # es_kwargs={"mirror_sampling": False}) + + emitter.ask()