Adding optimization specifically for RL (#1303)

* Adding optimization specifically for RL * fix * fix
facebookresearch · Dec 15, 2021 · de7efba · de7efba
1 parent 426a2a7
commit de7efba
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 13 deletions.
diff --git a/nevergrad/benchmark/gymexperiments.py b/nevergrad/benchmark/gymexperiments.py
@@ -110,21 +110,14 @@ def ng_full_gym(
 
     seedg = create_seed_generator(seed)
     optims = [
-        "CMA",
         "DiagonalCMA",
-        "OnePlusOne",
+        "GeneticDE",
+        "NoisyRL1",
+        "NoisyRL2",
+        "NoisyRL3",
+        "MixDeterministicRL",
+        "SpecialRL",
         "PSO",
-        "DiscreteOnePlusOne",
-        "DE",
-        "CMandAS2",
-        "NelderMead",
-        "DoubleFastGADiscreteOnePlusOne",
-        "DiscreteLenglerOnePlusOne",
-        "AnisotropicAdaptiveDiscreteOnePlusOne",
-        "TBPSA",
-        "SPSA",
-        "SQP",
-        "MetaModel",
     ]
     if multi:
         controls = ["multi_neural"]

diff --git a/nevergrad/optimization/experimentalvariants.py b/nevergrad/optimization/experimentalvariants.py
@@ -16,6 +16,13 @@
     NGOpt10,
     NGOpt12,
     BayesOptim,
+    ConfPortfolio,
+    DiagonalCMA,
+    GeneticDE,
+    TBPSA,
+    NoisyOnePlusOne,
+    RecombiningPortfolioOptimisticNoisyDiscreteOnePlusOne,
+    OptimisticNoisyOnePlusOne,
 )
 from . import optimizerlib as opts
 from .optimizerlib import CMA, Chaining, PSO, BO
@@ -328,3 +335,16 @@
 SparseDiscreteOnePlusOne = ParametrizedOnePlusOne(mutation="discrete", sparse=True).set_name(
     "SparseDiscreteOnePlusOne", register=True
 )
+
+# Specifically for RL.
+MixDeterministicRL = ConfPortfolio(optimizers=[DiagonalCMA, PSO, GeneticDE]).set_name(
+    "MixDeterministicRL", register=True
+)
+SpecialRL = Chaining([MixDeterministicRL, TBPSA], ["half"]).set_name("SpecialRL", register=True)
+NoisyRL1 = Chaining([MixDeterministicRL, NoisyOnePlusOne], ["half"]).set_name("NoisyRL1", register=True)
+NoisyRL2 = Chaining(
+    [MixDeterministicRL, RecombiningPortfolioOptimisticNoisyDiscreteOnePlusOne], ["half"]
+).set_name("NoisyRL2", register=True)
+NoisyRL3 = Chaining([MixDeterministicRL, OptimisticNoisyOnePlusOne], ["half"]).set_name(
+    "NoisyRL3", register=True
+)
diff --git a/nevergrad/optimization/recorded_recommendations.csv b/nevergrad/optimization/recorded_recommendations.csv
@@ -122,6 +122,7 @@ MilliCMA,0.0010125155,-0.0009138806,-0.0010295559,0.0012098418,,,,,,,,,,,,
 MiniDE,0.8273276988,-1.2921051963,-0.4797521288,0.2138608624,0.7088815721,0.7346249014,-2.6392592028,-1.0729615222,,,,,,,,
 MiniLhsDE,-0.0313128807,0.2738703026,-0.1988242191,0.9942001938,0.7167500893,-0.0350394443,-1.5341684983,-0.3039246928,,,,,,,,
 MiniQrDE,-0.2025746195,-0.8778768047,-1.2504657435,0.6265108481,0.4934247309,0.6448108695,-0.3573249779,-1.6986947217,,,,,,,,
+MixDeterministicRL,-0.3375952501,-0.585268278,-0.1149199408,2.2418177944,,,,,,,,,,,,
 MixES,1.1400386808,0.3380024444,0.4755144618,2.6390460807,0.6911075733,1.111235567,-0.2576843178,-1.1959512855,,,,,,,,
 MultiCMA,1.4855013085,-1.5953064496,-0.7871164493,-0.4908938162,,,,,,,,,,,,
 MultiDiscrete,0.0,0.0,0.0,1.095956118,,,,,,,,,,,,
@@ -157,6 +158,9 @@ NoisyDE,0.7325595717,-0.3250848292,-0.4968122173,1.9884218193,1.8577990761,1.772
 NoisyDiscreteOnePlusOne,0.7531428339,0.0,0.0,0.0,,,,,,,,,,,,
 NoisyInfSplits,0.0,0.0,0.0,0.0,,,,,,,,,,,,
 NoisyOnePlusOne,0.0,0.0,0.0,0.0,,,,,,,,,,,,
+NoisyRL1,-0.3375952501,-0.585268278,-0.1149199408,2.2418177944,,,,,,,,,,,,
+NoisyRL2,-0.3375952501,-0.585268278,-0.1149199408,2.2418177944,,,,,,,,,,,,
+NoisyRL3,-0.3375952501,-0.585268278,-0.1149199408,2.2418177944,,,,,,,,,,,,
 NonNSGAIIES,1.1400386808,0.3380024444,0.4755144618,2.6390460807,0.6911075733,1.111235567,-0.2576843178,-1.1959512855,,,,,,,,
 ORandomSearch,-0.4729858315,0.6814258794,-0.2424394967,1.700735634,,,,,,,,,,,,
 OScrHammersleySearch,-0.9674215661,0.0,0.4307272993,0.8416212336,,,,,,,,,,,,
@@ -207,6 +211,7 @@ ScrHammersleySearchPlusMiddlePoint,-1.2815515655,0.0,0.4307272993,0.8416212336,,
 Shiwa,0.0,-0.3451057176,-0.1327329683,1.9291307781,,,,,,,,,,,,
 SparseDiscreteOnePlusOne,0.7531428339,0.0,0.0,0.0,,,,,,,,,,,,
 SparseDoubleFastGADiscreteOnePlusOne,0.0,0.0,0.0,0.0942747145,,,,,,,,,,,,
+SpecialRL,-0.3375952501,-0.585268278,-0.1149199408,2.2418177944,,,,,,,,,,,,
 StupidRandom,-1.1543602352,-2.2133334794,-1.6817565104,-1.7880942511,,,,,,,,,,,,
 TBPSA,0.1302530513,0.3105038072,-0.0036907685,1.3766294785,1.1655103563,0.7923024939,-0.5540650904,-1.126716815,-0.4977202676,0.0718018969,,,,,,
 TEAvgCauchyLHSSearch,-0.527971877,1.341890246,2.6790716005,3.5963545262,,,,,,,,,,,,