From de7efba2a8185f1e603c61f949f4ab70e123143d Mon Sep 17 00:00:00 2001 From: Teytaud Date: Wed, 15 Dec 2021 10:25:05 +0100 Subject: [PATCH] Adding optimization specifically for RL (#1303) * Adding optimization specifically for RL * fix * fix --- nevergrad/benchmark/gymexperiments.py | 19 ++++++------------ .../optimization/experimentalvariants.py | 20 +++++++++++++++++++ .../optimization/recorded_recommendations.csv | 5 +++++ 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/nevergrad/benchmark/gymexperiments.py b/nevergrad/benchmark/gymexperiments.py index f489d7621..0d1efbd51 100644 --- a/nevergrad/benchmark/gymexperiments.py +++ b/nevergrad/benchmark/gymexperiments.py @@ -110,21 +110,14 @@ def ng_full_gym( seedg = create_seed_generator(seed) optims = [ - "CMA", "DiagonalCMA", - "OnePlusOne", + "GeneticDE", + "NoisyRL1", + "NoisyRL2", + "NoisyRL3", + "MixDeterministicRL", + "SpecialRL", "PSO", - "DiscreteOnePlusOne", - "DE", - "CMandAS2", - "NelderMead", - "DoubleFastGADiscreteOnePlusOne", - "DiscreteLenglerOnePlusOne", - "AnisotropicAdaptiveDiscreteOnePlusOne", - "TBPSA", - "SPSA", - "SQP", - "MetaModel", ] if multi: controls = ["multi_neural"] diff --git a/nevergrad/optimization/experimentalvariants.py b/nevergrad/optimization/experimentalvariants.py index d9628ab95..aab0633d8 100644 --- a/nevergrad/optimization/experimentalvariants.py +++ b/nevergrad/optimization/experimentalvariants.py @@ -16,6 +16,13 @@ NGOpt10, NGOpt12, BayesOptim, + ConfPortfolio, + DiagonalCMA, + GeneticDE, + TBPSA, + NoisyOnePlusOne, + RecombiningPortfolioOptimisticNoisyDiscreteOnePlusOne, + OptimisticNoisyOnePlusOne, ) from . import optimizerlib as opts from .optimizerlib import CMA, Chaining, PSO, BO @@ -328,3 +335,16 @@ SparseDiscreteOnePlusOne = ParametrizedOnePlusOne(mutation="discrete", sparse=True).set_name( "SparseDiscreteOnePlusOne", register=True ) + +# Specifically for RL. +MixDeterministicRL = ConfPortfolio(optimizers=[DiagonalCMA, PSO, GeneticDE]).set_name( + "MixDeterministicRL", register=True +) +SpecialRL = Chaining([MixDeterministicRL, TBPSA], ["half"]).set_name("SpecialRL", register=True) +NoisyRL1 = Chaining([MixDeterministicRL, NoisyOnePlusOne], ["half"]).set_name("NoisyRL1", register=True) +NoisyRL2 = Chaining( + [MixDeterministicRL, RecombiningPortfolioOptimisticNoisyDiscreteOnePlusOne], ["half"] +).set_name("NoisyRL2", register=True) +NoisyRL3 = Chaining([MixDeterministicRL, OptimisticNoisyOnePlusOne], ["half"]).set_name( + "NoisyRL3", register=True +) diff --git a/nevergrad/optimization/recorded_recommendations.csv b/nevergrad/optimization/recorded_recommendations.csv index 300ddd0df..8b97829dc 100644 --- a/nevergrad/optimization/recorded_recommendations.csv +++ b/nevergrad/optimization/recorded_recommendations.csv @@ -122,6 +122,7 @@ MilliCMA,0.0010125155,-0.0009138806,-0.0010295559,0.0012098418,,,,,,,,,,,, MiniDE,0.8273276988,-1.2921051963,-0.4797521288,0.2138608624,0.7088815721,0.7346249014,-2.6392592028,-1.0729615222,,,,,,,, MiniLhsDE,-0.0313128807,0.2738703026,-0.1988242191,0.9942001938,0.7167500893,-0.0350394443,-1.5341684983,-0.3039246928,,,,,,,, MiniQrDE,-0.2025746195,-0.8778768047,-1.2504657435,0.6265108481,0.4934247309,0.6448108695,-0.3573249779,-1.6986947217,,,,,,,, +MixDeterministicRL,-0.3375952501,-0.585268278,-0.1149199408,2.2418177944,,,,,,,,,,,, MixES,1.1400386808,0.3380024444,0.4755144618,2.6390460807,0.6911075733,1.111235567,-0.2576843178,-1.1959512855,,,,,,,, MultiCMA,1.4855013085,-1.5953064496,-0.7871164493,-0.4908938162,,,,,,,,,,,, MultiDiscrete,0.0,0.0,0.0,1.095956118,,,,,,,,,,,, @@ -157,6 +158,9 @@ NoisyDE,0.7325595717,-0.3250848292,-0.4968122173,1.9884218193,1.8577990761,1.772 NoisyDiscreteOnePlusOne,0.7531428339,0.0,0.0,0.0,,,,,,,,,,,, NoisyInfSplits,0.0,0.0,0.0,0.0,,,,,,,,,,,, NoisyOnePlusOne,0.0,0.0,0.0,0.0,,,,,,,,,,,, +NoisyRL1,-0.3375952501,-0.585268278,-0.1149199408,2.2418177944,,,,,,,,,,,, +NoisyRL2,-0.3375952501,-0.585268278,-0.1149199408,2.2418177944,,,,,,,,,,,, +NoisyRL3,-0.3375952501,-0.585268278,-0.1149199408,2.2418177944,,,,,,,,,,,, NonNSGAIIES,1.1400386808,0.3380024444,0.4755144618,2.6390460807,0.6911075733,1.111235567,-0.2576843178,-1.1959512855,,,,,,,, ORandomSearch,-0.4729858315,0.6814258794,-0.2424394967,1.700735634,,,,,,,,,,,, OScrHammersleySearch,-0.9674215661,0.0,0.4307272993,0.8416212336,,,,,,,,,,,, @@ -207,6 +211,7 @@ ScrHammersleySearchPlusMiddlePoint,-1.2815515655,0.0,0.4307272993,0.8416212336,, Shiwa,0.0,-0.3451057176,-0.1327329683,1.9291307781,,,,,,,,,,,, SparseDiscreteOnePlusOne,0.7531428339,0.0,0.0,0.0,,,,,,,,,,,, SparseDoubleFastGADiscreteOnePlusOne,0.0,0.0,0.0,0.0942747145,,,,,,,,,,,, +SpecialRL,-0.3375952501,-0.585268278,-0.1149199408,2.2418177944,,,,,,,,,,,, StupidRandom,-1.1543602352,-2.2133334794,-1.6817565104,-1.7880942511,,,,,,,,,,,, TBPSA,0.1302530513,0.3105038072,-0.0036907685,1.3766294785,1.1655103563,0.7923024939,-0.5540650904,-1.126716815,-0.4977202676,0.0718018969,,,,,, TEAvgCauchyLHSSearch,-0.527971877,1.341890246,2.6790716005,3.5963545262,,,,,,,,,,,,