diff --git a/bayes_opt/domain_reduction.py b/bayes_opt/domain_reduction.py index f4b86ffc8..40e9416f7 100644 --- a/bayes_opt/domain_reduction.py +++ b/bayes_opt/domain_reduction.py @@ -2,6 +2,7 @@ import numpy as np from .target_space import TargetSpace +from warnings import warn class DomainTransformer(): @@ -36,7 +37,10 @@ def __init__( self.minimum_window_value = minimum_window def initialize(self, target_space: TargetSpace) -> None: - """Initialize all of the parameters""" + """Initialize all of the parameters. + """ + + # Set the original bounds self.original_bounds = np.copy(target_space.bounds) self.bounds = [self.original_bounds] @@ -47,6 +51,7 @@ def initialize(self, target_space: TargetSpace) -> None: else: self.minimum_window = [self.minimum_window_value] * len(target_space.bounds) + # Set initial values self.previous_optimal = np.mean(target_space.bounds, axis=1) self.current_optimal = np.mean(target_space.bounds, axis=1) self.r = target_space.bounds[:, 1] - target_space.bounds[:, 0] @@ -72,14 +77,13 @@ def initialize(self, target_space: TargetSpace) -> None: self._window_bounds_compatibility(self.original_bounds) def _update(self, target_space: TargetSpace) -> None: - + """ Updates contraction rate, window size, and window center. + """ # setting the previous self.previous_optimal = self.current_optimal self.previous_d = self.current_d - - self.current_optimal = target_space.params[ - np.argmax(target_space.target) - ] + + self.current_optimal = target_space.params_to_array(target_space.max()['params']) self.current_d = 2.0 * (self.current_optimal - self.previous_optimal) / self.r @@ -97,32 +101,84 @@ def _update(self, target_space: TargetSpace) -> None: self.r = self.contraction_rate * self.r def _trim(self, new_bounds: np.array, global_bounds: np.array) -> np.array: - for i, variable in enumerate(new_bounds): - if variable[0] < global_bounds[i, 0]: - variable[0] = global_bounds[i, 0] - if variable[1] > global_bounds[i, 1]: - variable[1] = global_bounds[i, 1] - for i, entry in enumerate(new_bounds): - if entry[0] > entry[1]: - new_bounds[i, 0] = entry[1] - new_bounds[i, 1] = entry[0] - window_width = abs(entry[0] - entry[1]) - if window_width < self.minimum_window[i]: - dw = (self.minimum_window[i] - window_width) / 2.0 - left_expansion_space = abs(global_bounds[i, 0] - entry[0]) # should be non-positive - right_expansion_space = abs(global_bounds[i, 1] - entry[1]) # should be non-negative - # conservative - dw_l = min(dw, left_expansion_space) - dw_r = min(dw, right_expansion_space) - # this crawls towards the edge - ddw_r = dw_r + max(dw - dw_l, 0) - ddw_l = dw_l + max(dw - dw_r, 0) - new_bounds[i, 0] -= ddw_l - new_bounds[i, 1] += ddw_r + """ + Adjust the new_bounds and verify that they adhere to global_bounds and minimum_window. + + Parameters: + ----------- + new_bounds : np.array + The proposed new_bounds that (may) need adjustment. + + global_bounds : np.array + The maximum allowable bounds for each parameter. + + Returns: + -------- + new_bounds : np.array + The adjusted bounds after enforcing constraints. + """ + + #sort bounds + new_bounds = np.sort(new_bounds) + + # Validate each parameter's bounds against the global_bounds + for i, pbounds in enumerate(new_bounds): + # If the one of the bounds is outside the global bounds, reset the bound to the global bound + # This is expected to happen when the window is near the global bounds, no warning is issued + if (pbounds[0] < global_bounds[i, 0]): + pbounds[0] = global_bounds[i, 0] + + if (pbounds[1] > global_bounds[i, 1]): + pbounds[1] = global_bounds[i, 1] + + # If a lower bound is greater than the associated global upper bound, reset it to the global lower bound + if (pbounds[0] > global_bounds[i, 1]): + pbounds[0] = global_bounds[i, 0] + warn("\nDomain Reduction Warning:\n"+ + "A parameter's lower bound is greater than the global upper bound."+ + "The offensive boundary has been reset."+ + "Be cautious of subsequent reductions.", stacklevel=2) + + # If an upper bound is less than the associated global lower bound, reset it to the global upper bound + if (pbounds[1] < global_bounds[i, 0]): + pbounds[1] = global_bounds[i, 1] + warn("\nDomain Reduction Warning:\n"+ + "A parameter's lower bound is greater than the global upper bound."+ + "The offensive boundary has been reset."+ + "Be cautious of subsequent reductions.", stacklevel=2) + + # Adjust new_bounds to ensure they respect the minimum window width for each parameter + for i, pbounds in enumerate(new_bounds): + current_window_width = abs(pbounds[0] - pbounds[1]) + + # If the window width is less than the minimum allowable width, adjust it + # Note that when minimum_window < width of the global bounds one side always has more space than required + if current_window_width < self.minimum_window[i]: + width_deficit = (self.minimum_window[i] - current_window_width) / 2.0 + available_left_space = abs(global_bounds[i, 0] - pbounds[0]) + available_right_space = abs(global_bounds[i, 1] - pbounds[1]) + + # determine how much to expand on the left and right + expand_left = min(width_deficit, available_left_space) + expand_right = min(width_deficit, available_right_space) + + # calculate the deficit on each side + expand_left_deficit = width_deficit - expand_left + expand_right_deficit = width_deficit - expand_right + + # shift the deficit to the side with more space + adjust_left = expand_left + max(expand_right_deficit, 0) + adjust_right = expand_right + max(expand_left_deficit, 0) + + # adjust the bounds + pbounds[0] -= adjust_left + pbounds[1] += adjust_right + return new_bounds def _window_bounds_compatibility(self, global_bounds: np.array) -> bool: - """Checks if global bounds are compatible with the minimum window sizes.""" + """Checks if global bounds are compatible with the minimum window sizes. + """ for i, entry in enumerate(global_bounds): global_window_width = abs(entry[1] - entry[0]) if global_window_width < self.minimum_window[i]: @@ -133,7 +189,8 @@ def _create_bounds(self, parameters: dict, bounds: np.array) -> dict: return {param: bounds[i, :] for i, param in enumerate(parameters)} def transform(self, target_space: TargetSpace) -> dict: - + """Reduces the bounds of the target space. + """ self._update(target_space) new_bounds = np.array( @@ -143,6 +200,6 @@ def transform(self, target_space: TargetSpace) -> dict: ] ).T - self._trim(new_bounds, self.original_bounds) + new_bounds = self._trim(new_bounds, self.original_bounds) self.bounds.append(new_bounds) return self._create_bounds(target_space.keys, new_bounds) diff --git a/bayes_opt/target_space.py b/bayes_opt/target_space.py index e90cfbcf0..534932f2e 100644 --- a/bayes_opt/target_space.py +++ b/bayes_opt/target_space.py @@ -1,4 +1,5 @@ import numpy as np +from warnings import warn from .util import ensure_rng, NotUniqueError from .util import Colours @@ -19,9 +20,11 @@ class TargetSpace(object): >>> return p1 + p2 >>> pbounds = {'p1': (0, 1), 'p2': (1, 100)} >>> space = TargetSpace(target_func, pbounds, random_state=0) - >>> x = space.random_points(1)[0] - >>> y = space.register_point(x) - >>> assert self.max_point()['max_val'] == y + >>> x = np.array([4 , 5]) + >>> y = target_func(x) + >>> space.register(x, y) + >>> assert self.max()['target'] == 9 + >>> assert self.max()['params'] == {'p1': 1.0, 'p2': 2.0} """ def __init__(self, target_func, pbounds, constraint=None, random_state=None, @@ -116,6 +119,23 @@ def constraint_values(self): if self._constraint is not None: return self._constraint_values + @property + def mask(self): + '''Returns a boolean array of the points that satisfy the constraint and boundary conditions''' + mask = np.ones_like(self.target, dtype=bool) + + # mask points that don't satisfy the constraint + if self._constraint is not None: + mask &= self._constraint.allowed(self._constraint_values) + + # mask points that are outside the bounds + if self._bounds is not None: + within_bounds = np.all((self._bounds[:, 0] <= self._params) & + (self._params <= self._bounds[:, 1]), axis=1) + mask &= within_bounds + + return mask + def params_to_array(self, params): try: assert set(params) == set(self.keys) @@ -174,13 +194,14 @@ def register(self, params, target, constraint_value=None): Example ------- + >>> target_func = lambda p1, p2: p1 + p2 >>> pbounds = {'p1': (0, 1), 'p2': (1, 100)} - >>> space = TargetSpace(lambda p1, p2: p1 + p2, pbounds) + >>> space = TargetSpace(target_func, pbounds) >>> len(space) 0 >>> x = np.array([0, 0]) >>> y = 1 - >>> space.add_observation(x, y) + >>> space.register(x, y) >>> len(space) 1 """ @@ -194,6 +215,11 @@ def register(self, params, target, constraint_value=None): raise NotUniqueError(f'Data point {x} is not unique. You can set "allow_duplicate_points=True" to ' f'avoid this error') + # if x is not within the bounds of the parameter space, warn the user + if self._bounds is not None: + if not np.all((self._bounds[:, 0] <= x) & (x <= self._bounds[:, 1])): + warn(f'\nData point {x} is outside the bounds of the parameter space. ', stacklevel=2) + self._params = np.concatenate([self._params, x.reshape(1, -1)]) self._target = np.concatenate([self._target, [target]]) @@ -212,8 +238,8 @@ def register(self, params, target, constraint_value=None): def probe(self, params): """ - Evaluates a single point x, to obtain the value y and then records them - as observations. + Evaluates a single point x, to obtain the value y and then registers the + point and its evaluation. Notes ----- @@ -228,6 +254,15 @@ def probe(self, params): ------- y : float target function value. + + Example + ------- + >>> target_func = lambda p1, p2: p1 + p2 + >>> pbounds = {'p1': (0, 1), 'p2': (1, 100)} + >>> space = TargetSpace(target_func, pbounds) + >>> space.probe([1, 5]) + >>> assert self.max()['target'] == 6 + >>> assert self.max()['params'] == {'p1': 1.0, 'p2': 5.0} """ x = self._as_array(params) params = dict(zip(self._keys, x)) @@ -243,19 +278,19 @@ def probe(self, params): def random_sample(self): """ - Creates random points within the bounds of the space. + Creates a random point within the bounds of the space. Returns ---------- data: ndarray - [num x dim] array points with dimensions corresponding to `self._keys` + [1 x dim] array with dimensions corresponding to `self._keys` Example ------- >>> target_func = lambda p1, p2: p1 + p2 >>> pbounds = {'p1': (0, 1), 'p2': (1, 100)} >>> space = TargetSpace(target_func, pbounds, random_state=0) - >>> space.random_points(1) + >>> space.random_sample() array([[ 55.33253689, 0.54488318]]) """ data = np.empty((1, self.dim)) @@ -264,46 +299,49 @@ def random_sample(self): return data.ravel() def _target_max(self): - """Get maximum target value found. + """Get the maximum target value within the current parameter bounds. If there is a constraint present, the maximum value that fulfills the - constraint is returned.""" + constraint within the parameter bounds is returned. + + Returns + ---------- + max: float + The maximum target value. + + """ if len(self.target) == 0: return None - if self._constraint is None: - return self.target.max() - - allowed = self._constraint.allowed(self._constraint_values) - if allowed.any(): - return self.target[allowed].max() + if len(self.target[self.mask]) == 0: + return None - return None + return self.target[self.mask].max() def max(self): - """Get maximum target value found and corresponding parameters. - + """Get the maximum target value within the current parameter bounds, + and its associated parameters. + If there is a constraint present, the maximum value that fulfills the - constraint is returned.""" - target_max = self._target_max() + constraint within the parameter bounds is returned. + Returns + ---------- + res: dict + A dictionary with the keys 'target' and 'params'. The value of + 'target' is the maximum target value, and the value of 'params' is + a dictionary with the parameter names as keys and the parameter + values as values. + + """ + target_max = self._target_max() if target_max is None: return None - if self._constraint is not None: - allowed = self._constraint.allowed(self._constraint_values) - - target = self.target[allowed] - params = self.params[allowed] - constraint_values = self.constraint_values[allowed] - else: - target = self.target - params = self.params - constraint_values = self.constraint_values + target = self.target[self.mask] + params = self.params[self.mask] + target_max_idx = np.argmax(target) - target_max_idx = np.where(target == target_max)[0][0] - - res = { 'target': target_max, 'params': dict( @@ -312,13 +350,29 @@ def max(self): } if self._constraint is not None: + constraint_values = self.constraint_values[self.mask] res['constraint'] = constraint_values[target_max_idx] return res def res(self): """Get all target values and constraint fulfillment for all parameters. + + Returns + ---------- + res: list + A list of dictionaries with the keys 'target', 'params', and + 'constraint'. The value of 'target' is the target value, the value + of 'params' is a dictionary with the parameter names as keys and the + parameter values as values, and the value of 'constraint' is the + constraint fulfillment. + + Notes + ----- + Does not report if points are within the bounds of the parameter space. + """ + if self._constraint is None: params = [dict(zip(self.keys, p)) for p in self.params] diff --git a/tests/test_logs_bounds.log b/tests/test_logs_bounds.log new file mode 100644 index 000000000..0a7d9ba46 --- /dev/null +++ b/tests/test_logs_bounds.log @@ -0,0 +1,5 @@ +{"datetime": {"delta": 0.0, "datetime": "2018-11-25 08:29:25", "elapsed": 0.0}, "params": {"y": 0, "x": 0}, "target": 0} +{"datetime": {"delta": 0.001301, "datetime": "2018-11-25 08:29:25", "elapsed": 0.001301}, "params": {"y": 1, "x": 1}, "target": 2} +{"datetime": {"delta": 1.075242, "datetime": "2018-11-25 08:29:26", "elapsed": 1.076543}, "params": {"y": 2, "x": 2}, "target": 4} +{"datetime": {"delta": 0.239797, "datetime": "2018-11-25 08:29:26", "elapsed": 1.31634}, "params": {"y": 3, "x": 3}, "target": 6} +{"datetime": {"delta": 0.001301, "datetime": "2018-11-25 08:29:25", "elapsed": 0.001301}, "params": {"y": 1.5, "x": 1.5}, "target": 3} diff --git a/tests/test_seq_domain_red.py b/tests/test_seq_domain_red.py index 4b60bf049..f03dc71dc 100644 --- a/tests/test_seq_domain_red.py +++ b/tests/test_seq_domain_red.py @@ -68,6 +68,7 @@ def reset(self): assert not (standard_optimizer._space.bounds == mutated_optimizer._space.bounds).any() + def test_minimum_window_is_kept(): bounds_transformer = SequentialDomainReductionTransformer(minimum_window=1.0) pbounds = {'x': (-0.5, 0.5), 'y': (-1.0, 0.0)} @@ -106,6 +107,7 @@ def test_minimum_window_array_is_kept(): window_widths = np.diff(bounds_transformer.bounds) assert np.all(np.isclose(np.squeeze(np.min(window_widths, axis=0)), window_ranges)) + def test_trimming_bounds(): """Test if the bounds are trimmed correctly within the bounds""" def dummy_function(x1, x2, x3, x4, x5): @@ -143,4 +145,51 @@ def test_exceeded_bounds(): verbose=0, random_state=1, bounds_transformer=bounds_transformer - ) \ No newline at end of file + ) + + +def test_trim_when_both_new_bounds_exceed_global_bounds(): + """Test if the global bounds are respected when both new bounds for a given parameter + are beyond the global bounds.""" + + # initialize a bounds transformer + bounds_transformer = SequentialDomainReductionTransformer(minimum_window=10) + pbounds = {'x': (-10, 10),'y': (-10, 10)} + target_sp = TargetSpace(target_func=black_box_function, pbounds=pbounds) + bounds_transformer.initialize(target_sp) + global_bounds = np.asarray(list(pbounds.values())) + + def verify_bounds_in_range(new_bounds, global_bounds): + """Check if the new bounds are within the global bounds.""" + test = True + for i, pbounds in enumerate(new_bounds): + if (pbounds[0] < global_bounds[i, 0] or pbounds[0] > global_bounds[i, 1]): + test = False + if (pbounds[1] > global_bounds[i, 1] or pbounds[1] < global_bounds[i, 0]): + test = False + return test + + # test if the sorting of the bounds is correct + new_bounds = np.array( [[5, -5], [-10, 10]] ) + trimmed_bounds = bounds_transformer._trim(new_bounds, global_bounds) + assert (trimmed_bounds == np.array( [[-5, 5], [-10, 10]] )).all() + + # test if both (upper/lower) bounds for a parameter exceed the global bounds + new_bounds = np.array( [[-50, -20], [20, 50]] ) + with pytest.warns(UserWarning): + trimmed_bounds = bounds_transformer._trim(new_bounds, global_bounds) + assert verify_bounds_in_range(trimmed_bounds, global_bounds) + + # test if both (upper/lower) bounds for a parameter exceed the global bounds + # while they are out of order + new_bounds = np.array( [[-20, -50], [-10, 10]] ) + with pytest.warns(UserWarning): + trimmed_bounds = bounds_transformer._trim(new_bounds, global_bounds) + assert verify_bounds_in_range(trimmed_bounds, global_bounds) + +if __name__ == '__main__': + r""" + CommandLine: + python tests/test_seq_domain_red.py + """ + pytest.main([__file__]) \ No newline at end of file diff --git a/tests/test_target_space.py b/tests/test_target_space.py index c554dea0e..494b42495 100644 --- a/tests/test_target_space.py +++ b/tests/test_target_space.py @@ -74,6 +74,7 @@ def test_as_array(): def test_register(): + PBOUNDS = {'p1': (0, 10), 'p2': (1, 100)} space = TargetSpace(target_func, PBOUNDS) assert len(space) == 0 @@ -96,6 +97,7 @@ def test_register(): def test_register_with_constraint(): + PBOUNDS = {'p1': (0, 10), 'p2': (1, 100)} constraint = ConstraintModel(lambda x: x, -2, 2) space = TargetSpace(target_func, PBOUNDS, constraint=constraint) @@ -118,7 +120,16 @@ def test_register_with_constraint(): space.register(params={"p1": 2, "p2": 2}, target=3) +def test_register_point_beyond_bounds(): + PBOUNDS = {'p1': (0, 1), 'p2': (1, 10)} + space = TargetSpace(target_func, PBOUNDS) + + with pytest.warns(UserWarning): + space.register(params={"p1": 0.5, "p2": 20}, target=2.5) + + def test_probe(): + PBOUNDS = {'p1': (0, 10), 'p2': (1, 100)} space = TargetSpace(target_func, PBOUNDS, allow_duplicate_points=True) assert len(space) == 0 @@ -166,12 +177,14 @@ def test_random_sample(): def test_y_max(): space = TargetSpace(target_func, PBOUNDS) assert space._target_max() == None - space.probe(params={"p1": 1, "p2": 2}) - space.probe(params={"p1": 5, "p2": 1}) + space.probe(params={"p1": 1, "p2": 7}) + space.probe(params={"p1": 0.5, "p2": 1}) space.probe(params={"p1": 0, "p2": 1}) - assert space._target_max() == 6 + assert space._target_max() == 8 + def test_y_max_with_constraint(): + PBOUNDS = {'p1': (0, 10), 'p2': (1, 100)} constraint = ConstraintModel(lambda p1, p2: p1-p2, -2, 2) space = TargetSpace(target_func, PBOUNDS, constraint) assert space._target_max() == None @@ -181,10 +194,21 @@ def test_y_max_with_constraint(): assert space._target_max() == 3 +def test_y_max_within_pbounds(): + PBOUNDS = {'p1': (0, 2), 'p2': (1, 100)} + space = TargetSpace(target_func, PBOUNDS) + assert space._target_max() == None + space.probe(params={"p1": 1, "p2": 2}) + space.probe(params={"p1": 0, "p2": 1}) + with pytest.warns(UserWarning): + space.probe(params={"p1": 5, "p2": 1}) + assert space._target_max() == 3 + def test_max(): + PBOUNDS = {'p1': (0, 10), 'p2': (1, 100)} space = TargetSpace(target_func, PBOUNDS) - + assert space.max() == None space.probe(params={"p1": 1, "p2": 2}) space.probe(params={"p1": 5, "p2": 4}) @@ -192,7 +216,9 @@ def test_max(): space.probe(params={"p1": 1, "p2": 6}) assert space.max() == {"params": {"p1": 5, "p2": 4}, "target": 9} + def test_max_with_constraint(): + PBOUNDS = {'p1': (0, 10), 'p2': (1, 100)} constraint = ConstraintModel(lambda p1, p2: p1-p2, -2, 2) space = TargetSpace(target_func, PBOUNDS, constraint=constraint) @@ -203,7 +229,9 @@ def test_max_with_constraint(): space.probe(params={"p1": 1, "p2": 6}) # Unfeasible assert space.max() == {"params": {"p1": 2, "p2": 3}, "target": 5, "constraint": -1} + def test_max_with_constraint_identical_target_value(): + PBOUNDS = {'p1': (0, 10), 'p2': (1, 100)} constraint = ConstraintModel(lambda p1, p2: p1-p2, -2, 2) space = TargetSpace(target_func, PBOUNDS, constraint=constraint) @@ -215,7 +243,9 @@ def test_max_with_constraint_identical_target_value(): space.probe(params={"p1": 1, "p2": 6}) # Unfeasible assert space.max() == {"params": {"p1": 2, "p2": 3}, "target": 5, "constraint": -1} + def test_res(): + PBOUNDS = {'p1': (0, 10), 'p2': (1, 100)} space = TargetSpace(target_func, PBOUNDS) assert space.res() == [] diff --git a/tests/test_util.py b/tests/test_util.py index 16ec28fd6..5414b3566 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -132,7 +132,7 @@ def f(x, y): optimizer = BayesianOptimization( f=f, - pbounds={"x": (-2, 2), "y": (-2, 2)} + pbounds={"x": (-200, 200), "y": (-200, 200)} ) assert len(optimizer.space) == 0 @@ -150,6 +150,21 @@ def f(x, y): load_logs(other_optimizer, [str(test_dir / "test_logs.log")]) +def test_logs_bounds(): + def f(x, y): + return x + y + + optimizer = BayesianOptimization( + f=f, + pbounds={"x": (-2, 2), "y": (-2, 2)} + ) + + with pytest.warns(UserWarning): + load_logs(optimizer, [str(test_dir / "test_logs_bounds.log")]) + + assert len(optimizer.space) == 5 + + def test_logs_constraint(): def f(x, y): @@ -162,7 +177,7 @@ def c(x, y): optimizer = BayesianOptimization( f=f, - pbounds={"x": (-2, 2), "y": (-2, 2)}, + pbounds={"x": (-200, 200), "y": (-200, 200)}, constraint=constraint ) @@ -201,5 +216,4 @@ def test_colours(): CommandLine: python tests/test_target_space.py """ - import pytest - pytest.main([__file__]) + pytest.main([__file__]) \ No newline at end of file