From 01530a836cf989834bfbf0268a04650818e59a15 Mon Sep 17 00:00:00 2001 From: Jeremy Rapin Date: Mon, 18 Jan 2021 17:55:16 +0100 Subject: [PATCH 1/5] Make sure pruning does prune --- nevergrad/optimization/utils.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nevergrad/optimization/utils.py b/nevergrad/optimization/utils.py index 242b6f9f1..f62000336 100644 --- a/nevergrad/optimization/utils.py +++ b/nevergrad/optimization/utils.py @@ -258,8 +258,11 @@ def __init__(self, min_len: int, max_len: int): def __call__(self, archive: Archive[MultiValue]) -> Archive[MultiValue]: if len(archive) < self.max_len: return archive + return self._prune(archive) + + def _prune(self, archive: Archive[MultiValue]) -> Archive[MultiValue]: quantiles: tp.Dict[str, float] = {} - threshold = float(self.min_len) / len(archive) + threshold = float(self.min_len + 1) / len(archive) names = ["optimistic", "pessimistic", "average"] for name in names: quantiles[name] = np.quantile( @@ -269,8 +272,9 @@ def __call__(self, archive: Archive[MultiValue]) -> Archive[MultiValue]: new_archive.bytesdict = { b: v for b, v in archive.bytesdict.items() - if any(v.get_estimation(n) <= quantiles[n] for n in names) - } + if any(v.get_estimation(n) < quantiles[n] for n in names) + } # strict comparison to make sure we prune even for values repeated maaany times + # this may remove all points though, but nevermind for now return new_archive @classmethod From ad5002d105d1da031aaa6242fc4e105dadf73c9a Mon Sep 17 00:00:00 2001 From: Jeremy Rapin Date: Mon, 18 Jan 2021 17:57:41 +0100 Subject: [PATCH 2/5] comment --- nevergrad/optimization/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nevergrad/optimization/utils.py b/nevergrad/optimization/utils.py index f62000336..f1f87af28 100644 --- a/nevergrad/optimization/utils.py +++ b/nevergrad/optimization/utils.py @@ -261,6 +261,7 @@ def __call__(self, archive: Archive[MultiValue]) -> Archive[MultiValue]: return self._prune(archive) def _prune(self, archive: Archive[MultiValue]) -> Archive[MultiValue]: + # separate function to ease profiling quantiles: tp.Dict[str, float] = {} threshold = float(self.min_len + 1) / len(archive) names = ["optimistic", "pessimistic", "average"] From c7b193b42e65703df8676bef142332fba8c3931d Mon Sep 17 00:00:00 2001 From: Jeremy Rapin Date: Mon, 18 Jan 2021 18:16:04 +0100 Subject: [PATCH 3/5] test --- nevergrad/functions/test_base.py | 44 ++++++++++++++--------------- nevergrad/optimization/test_base.py | 14 +++++++++ nevergrad/optimization/utils.py | 2 ++ 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/nevergrad/functions/test_base.py b/nevergrad/functions/test_base.py index ce508eb5a..e6532cdf7 100644 --- a/nevergrad/functions/test_base.py +++ b/nevergrad/functions/test_base.py @@ -6,7 +6,7 @@ import pickle import numpy as np import pytest -from nevergrad.parametrization import parameter as p +import nevergrad as ng from nevergrad.common import testing from nevergrad.functions import ArtificialFunction import nevergrad.common.typing as tp @@ -20,12 +20,12 @@ def _arg_return(*args: tp.Any, **kwargs: tp.Any) -> float: def test_experiment_function() -> None: - param = p.Instrumentation( - p.Choice([1, 12]), + param = ng.p.Instrumentation( + ng.p.Choice([1, 12]), "constant", - p.Array(shape=(2, 2)), + ng.p.Array(shape=(2, 2)), constkwarg="blublu", - plop=p.Choice([3, 4]), + plop=ng.p.Choice([3, 4]), ) with pytest.raises(RuntimeError): base.ExperimentFunction(_arg_return, param) @@ -53,11 +53,11 @@ def test_experiment_function() -> None: def test_instrumented_function_kwarg_order() -> None: ifunc = base.ExperimentFunction( _arg_return, - p.Instrumentation( - kw4=p.Choice([1, 0]), + ng.p.Instrumentation( + kw4=ng.p.Choice([1, 0]), kw2="constant", - kw3=p.Array(shape=(2, 2)), - kw1=p.Scalar(2.0).set_mutation(sigma=2.0), + kw3=ng.p.Array(shape=(2, 2)), + kw1=ng.p.Scalar(2.0).set_mutation(sigma=2.0), ).set_name("test"), ) np.testing.assert_equal(ifunc.dimension, 7) @@ -74,16 +74,16 @@ def __call__(self, x: float, y: float = 0) -> float: def test_callable_parametrization() -> None: - ifunc = base.ExperimentFunction(lambda x: x ** 2, p.Scalar(2).set_mutation(2).set_name("")) # type: ignore + ifunc = base.ExperimentFunction(lambda x: x ** 2, ng.p.Scalar(2).set_mutation(2).set_name("")) # type: ignore np.testing.assert_equal(ifunc.descriptors["name"], "") - ifunc = base.ExperimentFunction(_Callable(), p.Scalar(2).set_mutation(sigma=2).set_name("")) + ifunc = base.ExperimentFunction(_Callable(), ng.p.Scalar(2).set_mutation(sigma=2).set_name("")) np.testing.assert_equal(ifunc.descriptors["name"], "_Callable") # test automatic filling assert len(ifunc._auto_init) == 2 def test_packed_function() -> None: - ifunc = base.ExperimentFunction(_Callable(), p.Scalar(1).set_name("")) + ifunc = base.ExperimentFunction(_Callable(), ng.p.Scalar(1).set_name("")) with pytest.raises(AssertionError): base.MultiExperiment([ifunc, ifunc], [100, 100]) pfunc = base.MultiExperiment([ifunc, ifunc.copy()], [100, 100]) @@ -92,7 +92,7 @@ def test_packed_function() -> None: def test_deterministic_data_setter() -> None: - instru = p.Instrumentation(p.Choice([0, 1, 2, 3]), y=p.Choice([0, 1, 2, 3])).set_name("") + instru = ng.p.Instrumentation(ng.p.Choice([0, 1, 2, 3]), y=ng.p.Choice([0, 1, 2, 3])).set_name("") ifunc = base.ExperimentFunction(_Callable(), instru) data = [0.01, 0, 0, 0, 0.01, 0, 0, 0] for _ in range(20): @@ -113,20 +113,20 @@ def test_deterministic_data_setter() -> None: @testing.parametrized( - floats=((p.Scalar(), p.Scalar(init=12.0)), True, False), - array_int=((p.Scalar(), p.Array(shape=(1,)).set_integer_casting()), False, False), - softmax_noisy=((p.Choice(["blue", "red"]), p.Array(shape=(1,))), True, True), + floats=((ng.p.Scalar(), ng.p.Scalar(init=12.0)), True, False), + array_int=((ng.p.Scalar(), ng.p.Array(shape=(1,)).set_integer_casting()), False, False), + softmax_noisy=((ng.p.Choice(["blue", "red"]), ng.p.Array(shape=(1,))), True, True), softmax_deterministic=( - (p.Choice(["blue", "red"], deterministic=True), p.Array(shape=(1,))), + (ng.p.Choice(["blue", "red"], deterministic=True), ng.p.Array(shape=(1,))), False, False, ), - ordered_discrete=((p.TransitionChoice([True, False]), p.Array(shape=(1,))), False, False), + ordered_discrete=((ng.p.TransitionChoice([True, False]), ng.p.Array(shape=(1,))), False, False), ) def test_parametrization_continuous_noisy( - variables: tp.Tuple[p.Parameter, ...], continuous: bool, noisy: bool + variables: tp.Tuple[ng.p.Parameter, ...], continuous: bool, noisy: bool ) -> None: - instru = p.Instrumentation(*variables) + instru = ng.p.Instrumentation(*variables) assert instru.descriptors.continuous == continuous assert instru.descriptors.deterministic != noisy @@ -134,7 +134,7 @@ def test_parametrization_continuous_noisy( class ExampleFunction(base.ExperimentFunction): def __init__(self, dimension: int, number: int, default: int = 12): # pylint: disable=unused-argument # unused argument is used to check that it is automatically added as descriptor - super().__init__(self.oracle_call, p.Array(shape=(dimension,))) + super().__init__(self.oracle_call, ng.p.Array(shape=(dimension,))) def oracle_call(self, x: np.ndarray) -> float: return float(x[0]) @@ -157,7 +157,7 @@ def test_function_descriptors_and_pickle() -> None: class ExampleFunctionAllDefault(base.ExperimentFunction): def __init__(self, dimension: int = 2, default: int = 12): # pylint: disable=unused-argument # unused argument is used to check that it is automatically added as descriptor - super().__init__(lambda x: 3.0, p.Array(shape=(dimension,))) + super().__init__(lambda x: 3.0, ng.p.Array(shape=(dimension,))) def test_function_descriptors_all_default() -> None: diff --git a/nevergrad/optimization/test_base.py b/nevergrad/optimization/test_base.py index 8731b23e1..73f6190ab 100644 --- a/nevergrad/optimization/test_base.py +++ b/nevergrad/optimization/test_base.py @@ -12,6 +12,7 @@ from . import optimizerlib from . import experimentalvariants as xpvariants from . import base +from . import utils from . import callbacks @@ -177,3 +178,16 @@ def test_recommendation_correct() -> None: optimizer = optimizerlib.OnePlusOne(parametrization=param, budget=300, num_workers=1) recommendation = optimizer.minimize(func) assert func.min_loss == recommendation.value + + +def constant(x: np.ndarray) -> float: # pylint: disable=unused-argument + return 12.0 + + +def test_pruning_calls() -> None: + opt = ng.optimizers.CMA(50, budget=2000) + # worst case scenario for pruning is constant: + # it should not keep everything or that will make computation time explode + opt.minimize(constant) + assert isinstance(opt.pruning, utils.Pruning) + assert opt.pruning._num_prunings < 4 diff --git a/nevergrad/optimization/utils.py b/nevergrad/optimization/utils.py index f1f87af28..6f22c375c 100644 --- a/nevergrad/optimization/utils.py +++ b/nevergrad/optimization/utils.py @@ -254,6 +254,7 @@ class Pruning: def __init__(self, min_len: int, max_len: int): self.min_len = min_len self.max_len = max_len + self._num_prunings = 0 # for testing it is not called too often def __call__(self, archive: Archive[MultiValue]) -> Archive[MultiValue]: if len(archive) < self.max_len: @@ -261,6 +262,7 @@ def __call__(self, archive: Archive[MultiValue]) -> Archive[MultiValue]: return self._prune(archive) def _prune(self, archive: Archive[MultiValue]) -> Archive[MultiValue]: + self._num_prunings += 1 # separate function to ease profiling quantiles: tp.Dict[str, float] = {} threshold = float(self.min_len + 1) / len(archive) From 470e34a9033a15aa85d456ecebaf0cd321590b3f Mon Sep 17 00:00:00 2001 From: Jeremy Rapin Date: Mon, 18 Jan 2021 18:40:08 +0100 Subject: [PATCH 4/5] changeversion --- nevergrad/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nevergrad/__init__.py b/nevergrad/__init__.py index a8774566b..1b2c1074c 100644 --- a/nevergrad/__init__.py +++ b/nevergrad/__init__.py @@ -13,4 +13,4 @@ __all__ = ["optimizers", "families", "callbacks", "p", "typing"] -__version__ = "0.4.2.post5" +__version__ = "0.4.2.post6" From 777ea637bdb11b4aa920bcc2d0b15674c4e3ef67 Mon Sep 17 00:00:00 2001 From: Jeremy Rapin Date: Mon, 18 Jan 2021 18:55:42 +0100 Subject: [PATCH 5/5] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e7ac8b9ec..2b44992d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - from now on, code formatting needs to be [`black`](https://black.readthedocs.io/en/stable/) compliant. This is simply performed by running `black nevergrad`. A continuous integration checks that PRs are compliant, and the precommit hooks have been adapted. For PRs branching from an old master, you can run `black --line-length=110 nevergrad/` to make your code easier to merge. +- Pruning has been patched to make sure it is not activated too often upon convergence [#1014](https://github.com/facebookresearch/nevergrad/pull/1014). The bug used to lead to important slowdown when reaching near convergence. ## 0.4.2 (2020-08-04)