Fix, and adding new algorithms, including NgIoh21 which might be the …

…new best black-box optimization Wizard (#1587) * fix * fix * tentative * fix * fix * fix * fix * black * fix * fix * static * newbfgs * alotofstuffwithngioh * fix * po * betterfix * fix * remove_useless_mess * fix * yetanother * refix * refix * fix * fix * fix * black_and_fix * fixblack * fix * fix * black_and_fix * black_and_fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix * fixtest * fix * addreco
facebookresearch · Jan 16, 2024 · b077302 · b077302
1 parent aab2282
commit b077302
Show file tree

Hide file tree

Showing 9 changed files with 1,874 additions and 82 deletions.
diff --git a/nevergrad/benchmark/experiments.py b/nevergrad/benchmark/experiments.py
@@ -49,7 +49,90 @@
 # pylint: disable=stop-iteration-return, too-many-nested-blocks, too-many-locals
 
 
+#    list_optims = ["QOTPDE", "LQOTPDE", "LQODE", "BAR4", "NGOpt", "CMandAS2"]
+#    list_optims = ["QOTPDE", "LQOTPDE", "LQODE"]
+#    list_optims = ["SPQODE", "SQOPSO", "DiagonalCMA"]
 def refactor_optims(x: tp.List[tp.Any]) -> tp.List[tp.Any]:  # type: ignore
+    #    return [np.random.choice(["ChainCMASQP", "BFGSCMAPlus", "SQPCMAPlus", "ChainMetaModelSQP", "QNDE", "NGOptRW", "MultiCobyla", "LBFGSB", "NLOPT-LN-COBYLA", "MultiCobylaPlus", "RCobyla", "PymooBIPOP", "MultiSQP", "MultiBFGS", "LogBFGSCMAPlus", "RBFGS", "SQP", "LogSQPCMAPlus", "MultiSQPPlus", "RSQP", "MultiBFGSPlus", "BFGS", "LogMultiBFGS", "SqrtMultiBFGS", "F3SQPCMA", "SqrtSQPCMAPlus", "LogMultiBFGSPlus", "SqrtBFGSCMAPlus", "TinySQP", "MicroSQP", "SqrtMultiBFGSPlus" ])]
+    # if np.random.randn() < 0.0:
+    #    return ["NgIoh8"]
+    return ["DE", "NGOpt", "NgIoh11", "RandomSearch"]
+    if False:  # np.random.randn() < 0.0:
+        return list(
+            np.random.choice(
+                [
+                    "NgIoh4",
+                    "NGOpt",
+                    "NGOptRW",
+                    "ChainCMASQP",
+                    "PymooBIPOP",
+                    "NLOPT_LN_SBPLX",
+                    "QNDE",
+                    "BFGSCMAPlus",
+                    "ChainMetaModelSQP",
+                    "BFGSCMA",
+                    "BAR4",
+                    "BFGSCMAPlus",
+                    "LBFGSB",
+                    "LQOTPDE",
+                    "LogSQPCMA",
+                ],
+                4,
+            )
+        )
+    return [
+        "CMandAS2",
+        "SQOPSO",
+        "SPQODE",
+        "DiagonalCMA",
+        "QOTPDE",
+        "LQOTPDE",
+        "LQODE",
+        "BAR4",
+        "ChainCMASQP",
+        "TinySQP",
+        "MicroSQP",
+        "MultiBFGSPlus",
+        "LogMultiBFGSPlus",
+        "PymooBIPOP",
+        "SqrtMultiBFGSPlus",
+        "MultiCobylaPlus",
+        "MultiSQPPlus",
+        "BFGSCMAPlus",
+        "LogBFGSCMAPlus",
+        "SqrtBFGSCMAPlus",
+        "SQPCMAPlus",
+        "LogSQPCMAPlus",
+        "SqrtSQPCMAPlus",
+        "MultiBFGS",
+        "LogMultiBFGS",
+        "SqrtMultiBFGS",
+        "MultiCobyla",
+        "MultiSQP",
+        "BFGSCMA",
+        "LogBFGSCMA",
+        "SqrtBFGSCMA",
+        "SQPCMA",
+        "LogSQPCMA",
+        "SqrtSQPCMA",
+        "FSQPCMA",
+        "F2SQPCMA",
+        "F3SQPCMA",
+        "ChainMetaModelSQP",
+        "BFGS",
+        "RBFGS",
+        "LBFGSB",
+        "Cobyla",
+        "RCobyla",
+        "SQP",
+        "RSQP",
+        "NLOPT_LN_COBYLA",
+        "NgIoh4",
+        "CMA",
+        "NGOpt",
+        "NGOptRW",
+        "QNDE",
+    ]
     # return ["RandomSearch", "OnePlusOne", "DE", "PSO"]
     list_optims = x
     algos = {}
@@ -411,7 +494,7 @@ def refactor_optims(x: tp.List[tp.Any]) -> tp.List[tp.Any]:  # type: ignore
     benchmark = str(inspect.stack()[1].function)
     if benchmark in algos and "tunin" in benchmark and np.random.randint(2) > 0 and False:
         return algos[benchmark][:5]
-
+    return ["NGOpt", "NgIoh17"]
     # Here, we pseudo-randomly draw one optim in the provided list,
     # depending on the host (so that each host is using the same optim).
     #    list_optims = x
@@ -519,7 +602,6 @@ def doint(s):  # Converting a string into an int.
             ]
             if ("Smooth" in a or "Lognor" in a or "Recomb" in a)
         ]
-    list_optims = ["NgIoh7", "NgIohRW2", "LognormalDiscreteOnePlusOne"]
 
     return [list_optims[doint(host) % len(list_optims)]]
 

diff --git a/nevergrad/functions/leaderboard.csv b/nevergrad/functions/leaderboard.csv
@@ -7,20 +7,20 @@
 "arcoating,70,1000",16.343215903293,"[5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0,5.0]"
 "arcoating,8,400",13.100717405397,"[5.0,6.333333333333333,7.666666666666666,9.0,9.0,9.0,9.0,9.0]"
 "bragg,16",0.932163816851,"[2.5,2.5,2.5,2.5,2.5,2.5,2.5,2.5,2.5,2.5,2.5,2.5,2.5,2.5,2.5,2.5]"
-"bragg,20",0.262874280769,"[2.9998300671577454,2.000316128920531,2.999863862991333,2.0001435222948203,2.999988317489624,2.000103992984805,2.9998511075973506,2.0001372592814732,2.9999333620071407,2.0000103827378553,86.84956312179565,105.51233768463135,85.7672068476677,107.91299521923065,87.41589814424515,104.85804855823517,86.86930418014526,105.44565796852112,86.41162544488907,106.38719379901886]"
+"bragg,20",0.262660386846,"[3.0,2.0,3.0,2.0,3.0,2.0,3.0,2.0,3.0,2.0,86.60251396782124,106.06603294036326,86.60257715690922,106.06602134614234,86.60252390427492,106.06600685908609,86.60251844028384,106.06603442407584,86.60256896015298,106.06593411074232]"
 "bragg,40",0.060042569169,"[3.0,2.0,3.0,2.0,2.9921018727896174,2.556185653707756,2.0,3.0,2.0,3.0,2.0,3.0,2.0,2.78089812089261,3.0,2.0,3.0,2.0,3.0,2.0,81.85048648120221,103.40845081696078,91.03041653572016,100.89528443524628,104.3789260959985,167.89870773566335,116.69132888282667,82.91546169078914,107.12477772036188,87.30165748774004,105.99606480281693,85.44843088843211,107.68209478044534,180.0,90.79404105608135,107.26309208762166,85.10052110978467,102.76832976413321,86.35727350241352,107.02623793814124]"
 "bragg,80",0.000694261986,"[2.999999999999995,2.0,2.9999999999999982,2.0,2.9999999999999942,2.0000000000000204,2.9999999999999982,2.0000000000000204,2.999999999999997,2.0000000000000577,2.999999999999994,2.0000000000000178,2.9999999999999996,2.0000000000000258,2.9999999999999956,2.0,3.0,2.0,3.0,2.0000000000000258,2.9999999999998677,2.0,2.999999999999998,2.0000000000000524,2.999999999999998,2.0,2.999999999999731,2.0000000000000204,3.0,2.0,3.0,2.0,2.999999999999869,2.0000000000000524,2.999999999999998,2.0,2.999999999999731,2.0,2.9999999999999982,2.0,86.60255045244762,106.06599965930195,86.60255443305651,106.06600747129924,86.60257272790835,106.06594450536937,86.60261007043326,106.06596664220048,86.60256991846003,106.06601083817202,86.60251126812021,106.06605239950612,86.60252027566017,106.06598075034444,86.60259101280616,106.06599907105759,86.60256520284221,106.06597421940252,86.60257498882078,106.06598483902887,86.60258117175987,106.06597850626063,86.60257221703684,106.06597517269579,86.60257273903393,106.06600072357362,86.60255405767539,106.06597375534662,86.60256934296459,106.06596027660167,86.60259348834224,106.06598724294983,86.60256638431304,106.06597523757073,86.60257261435468,106.06600050328808,86.60255444923789,106.06596589510059,86.60257650073991,106.06599053476317]"
-"bragg_as_tuple,20",0.001055185436,"[2.96990658846803,179.99966546149128,2.998902264319352,30.000241270329738,2.8265591075008514,179.99855339763468,2.0330863007122586,30.0075468811403,2.729187129693892,179.99997601273367,2.0864372478250535,101.79937826535131,2.157987763145396,179.9731762612071,2.9999305257329834,169.45231739088968,2.990478427835228,131.29092080107694,2.0000209948939642,168.1123248736521]"
+"bragg_as_tuple,20",0.000967522604,"[2.7384180752761087,179.9857733127887,2.447031109228277,30.00255899461493,2.0327446332476016,179.89597563026103,2.080675998280727,30.000344030422454,2.040745681951657,179.99999163508858,2.288196760615292,123.27386237699338,2.047571445661079,76.48484619105909,2.8497153563088053,57.08502183685728,2.9411143089422156,75.61181647086721,2.792767323107568,56.54220553571043]"
 "bragg_as_tuple,40",5.5972377e-05,"[2.948532392340755,180.0,3.0,30.0,3.0,131.06741701601484,2.767549191280395,30.5420401371438,3.0,179.65393533792871,2.7442402300284474,30.0,2.0966015662381756,161.99141733419847,2.953488425364594,30.0,2.016551585806329,180.0,3.0,179.99999998383626,2.292308532321335,32.51293698396886,2.189400571019735,180.0,2.149885114318265,121.61754339742575,2.0077928484103507,179.95644814413401,2.9999994034898143,102.23814079927654,3.0,180.0,3.0,180.0,2.999999840775769,180.0,2.031428198020951,180.0,2.9992244152448473,123.81355946097594]"
 "bragg_as_tuple,80",2.45365e-07,"[2.9549331072639724,158.75621656323568,2.0480044357819165,30.0,2.0,55.27628613454472,3.0,67.23237818684427,3.0,30.0,2.9999989607387887,180.0,3.0,30.0,2.0,180.0,2.0,30.0,3.0,119.67526617340124,3.0,30.0,3.0,179.99990709110386,2.0,30.0,2.242564929138231,180.0,2.0000000000001,30.0,2.9995932244962082,47.56425920994913,2.0,30.0,2.0,180.0,2.0,30.00714481618614,2.9848087851737124,150.86140143154492,2.0130071645286125,180.0,3.0,180.0,2.4383203911775704,30.0,2.9998437592268403,30.00062024271962,2.0,30.0,2.121112606913359,30.000000000804718,3.0,30.0,3.0,30.0,3.0,30.0,2.974233005367781,180.0,2.707185961252868,180.0,2.0,123.13187888743329,2.0000004584930373,81.98162132924631,2.9975818344303513,30.0,2.0,180.0,3.0,30.0,2.0,30.0,3.0,144.52320172542886,3.0,30.0,2.0,179.99993209998826]"
 "cf_photosic_realistic,20",0.002995472197,"[1.5128230497405948,3.717154614472663,3.707045125039246,7.5934554974188195,5.896974832265306,5.0537538791291094,2.7758528752281246,1.1078452822457434,2.7350705682170715,8.928402887071964,108.3639502018475,32.955014903470584,36.16917571098814,97.06106340305912,165.0279332123102,168.4577230968159,94.92443772322517,51.09011681229367,102.22825740735775,44.31565860858702]"
 "cf_photosic_realistic,40",0.006834812177,"[1.8448076337942485,1.5752964234856326,2.9031806811402845,4.274856113986125,5.930077195785369,5.118296006594735,4.5661298813336115,5.336743015564694,4.8751004072893025,3.3365200126208445,2.3808103158606047,1.778089786945824,2.4152195567556576,1.6776162044329448,3.618436080840265,6.795642693396257,6.182464813823576,5.0328064801025585,2.621393263573584,7.61214056900606,40.50110813461771,35.15466194674434,30.66683659316608,47.86180197526095,167.6582645007217,42.30938245971965,91.69534538260075,41.35380130989944,64.27253585507262,61.78587741879744,58.52113117825871,171.73235033539623,144.14783191364023,174.52584533984646,63.57125873977025,113.59466811951472,36.93988807988892,48.15876779055445,162.1563880600043,45.57703788786838]"
 "cf_photosic_realistic,80",0.0117040149799999,"[1.5354759424912032,3.7634283854949064,7.78055303945254,5.419009506164515,5.887505417790904,5.300327441401545,4.693285717949118,5.276400402425426,4.147953513561486,5.442226639500226,5.058313416220825,6.329593225605656,5.309365878301927,5.357638072367861,5.404854028026162,6.454853724025947,5.199665376183837,5.324646998652084,4.678629289422255,4.782255897667831,4.487306261881625,4.400458418783036,4.326756334112336,4.548763117259155,4.587200893804092,4.883841853734641,4.603366853105259,5.3918143230702835,4.84972133207243,4.686226787052884,5.2537087624393415,4.065915062434549,6.038037463134169,3.8776073040942354,5.606534847884856,4.859561195312973,4.447980485127308,4.946818321073501,4.052893773250138,8.080498848247768,114.35575173077771,67.18182512728241,95.34537711525937,99.12046541353337,98.1765822651763,73.36892584266718,119.20393211851366,98.81014955886698,120.39017966600399,89.06991537834554,86.29833751834202,95.80948667011639,85.69154339261523,65.07678408915123,80.69654401384784,96.33240855344756,83.11648895958243,105.62899287533395,122.53605144314626,94.81006597422127,100.85484935205464,101.51520718936922,96.8402283546034,110.60545941550778,69.46675675126359,94.93649649931615,100.63181845019137,109.88754207089882,123.44771989877177,98.23001404277618,93.4252830975658,114.80007609127351,98.76244219146753,125.53766468122589,105.26360561494576,94.67865828463431,121.14819160950388,78.22860163450872,120.72917887507884,44.409759060224054]"
-"cf_photosic_realistic_as_tuple,20",0.334494007114,"[1.0,30.0,9.0,180.0,2.858320140358255,107.17809855580609,9.0,69.73776585312172,9.0,30.001713367765745,1.0,30.0,1.0,40.522462581206085,1.0,180.0,1.0150457746296468,180.0,1.0,30.0]"
+"cf_photosic_realistic_as_tuple,20",0.334431744452,"[1.0,30.0,8.791004469187339,180.0,9.0,38.7617818690195,1.0,30.0,3.8647653702684632,30.0,9.0,30.0,1.0,37.83698027679809,1.0005665447308525,30.0,9.0,30.0,1.0,153.14417228219187]"
 "cf_photosic_realistic_as_tuple,40",0.348172807944,"[1.0016922423071986,30.077992408849965,8.996212944794262,173.62108466874264,9.0,39.11309181551697,8.972365568065815,30.772436561525893,1.5285333768055782,30.000198836608917,3.5626235577354253,31.582563789359682,2.709344855458358,30.001112055765105,1.3190462866510155,30.39185869049585,1.2129765787319151,38.68841939012668,1.0000143886280801,30.00000044034563,8.368891217008336,30.00000000000061,1.0078842676334916,38.504478451856215,1.0000000015059185,30.019317610268388,8.999181412786786,30.000017931320897,2.2280626868846514,30.0002005920108,1.0277153512797597,30.00000000057625,1.0045818032112317,30.000635312294676,8.816749567564345,154.33909209731098,1.042655780051407,30.00526784492037,1.0183047119242734,147.9313838866714]"
 "cf_photosic_realistic_as_tuple,80",0.404169772909,"[9.0,30.0,1.0004533379699811,153.81646404402713,9.0,78.72937421676897,5.9438720375770595,67.3132266615084,1.1217971357425833,98.98758721772973,1.0,86.4013458172307,1.522779937944145,124.7981352984252,1.0,30.0,1.0306189608156928,30.0,1.1868625360678497,167.06791550951903,9.0,180.0,1.0,180.0,9.0,180.0,8.22024890870132,180.0,1.0,180.0,1.0,180.0,1.0000145660471578,112.15013346953317,7.982797290869866,63.902010487372685,1.0051665533253256,46.238094022127996,2.0881739185199404,30.0,1.0,30.0,1.0,160.84225048904779,1.0,180.0,1.0,81.458388701988,1.7381504155696645,179.99917387231125,1.0,123.19052829814461,2.2901196519685083,30.000271428929608,1.0,32.129323023030466,9.0,30.0,1.0000003231782753,31.747439327107273,1.0,180.0,1.04681878490213,30.91215521011327,1.000000012368941,180.0,1.0,180.0,1.0000000000000004,139.9651585891046,1.0,168.42874914307842,1.0559553354291356,44.86481390371662,1.0000228988083206,180.0,1.0,30.0,1.0,30.0]"
 "cf_photosic_reference,16",0.431072337632,"[109.72985831489979,98.18574120560555,107.42439496690479,87.99264365961669,112.531428339492,89.65278659791956,105.05127078132273,103.79772329843809,96.93018121427048,133.71819394988918,99.02177080593655,109.72456993535758,115.95956118008563,92.84831198907784,118.42356371437981,103.77850212331678]"
-"cf_photosic_reference,20",0.082843553147,"[84.60641384124756,93.32310140132904,160.6474506855011,118.69610130786896,149.33961510658264,120.1955097913742,143.7487918138504,121.4040058851242,141.8354469537735,120.70926904678345,142.6336634159088,119.34617042541504,145.00528156757355,117.46501207351685,144.83027637004852,122.64509081840515,133.309428691864,130.55277943611145,91.41407936811447,74.78745460510254]"
+"cf_photosic_reference,20",0.082843547856,"[84.60239618871009,93.33412405133919,160.63670428542548,118.70556868589199,149.33737955560142,120.18837085408515,143.7538376975249,121.41728007831863,141.8077205278171,120.71488175843417,142.64760736365787,119.3383081113174,145.00589986129506,117.46103468845882,144.83611334892055,122.6437147953253,133.31203509894505,130.54758588238965,91.42140395577105,74.7852958204411]"
 "cf_photosic_reference,40",0.084939213136,"[81.39742494198082,101.44119728571083,158.0140104960585,121.6790534278067,145.81503548139966,122.09136794482608,143.2159642351609,120.28581189218119,143.68590105131614,123.17515218812616,135.78132612127558,121.49622264626781,140.50868649530034,119.10617900712968,142.73884544725277,118.05933430040943,141.68861960213334,118.2153522246694,140.29356028063077,118.12015221425766,142.80319825316133,119.70336315420906,141.9642808224119,120.41009879300351,148.86542372088152,119.20766379192047,156.74457524664513,128.2412139084457,180.0,151.44913736453114,180.0,134.311573138848,159.5878954063337,117.87077801701192,156.02745207576675,119.08396522657013,140.79077419270777,130.885157037691,90.38456262594073,75.81598303665277]"
 "cf_photosic_reference,80",0.165955044925,"[125.4884376485969,31.43727941883506,176.81212680274388,163.64673359279863,30.0,30.0,30.0,71.56336730735535,30.0,30.0,60.132281235779,175.63036573239486,180.0,31.23975421957407,46.60278764515358,52.22319313678755,30.0,33.95360490313314,30.0,30.0,34.81114395503347,175.37126668418534,178.30316182499982,179.03557619632215,43.038612007695384,175.26482763106216,174.69345429657795,30.0,32.87217264940138,170.58395850009873,30.577092835567512,30.0,98.28872399072498,32.897365327420765,34.67421888315457,95.94717292400826,176.54174484362667,158.96194700502335,180.0,56.8021373584387,30.0,67.51086610415719,30.0,168.37571929336832,163.1834829467999,122.01267648173253,180.0,174.71085052869373,30.0,166.82793878846667,146.45048983362932,170.78674243606304,179.49135953574967,53.97384420929961,35.67944571001503,43.75630407024304,41.21954307574285,30.407798095659107,35.183188344660245,118.29440868693369,30.495811828657,176.3255931763546,175.70714392505408,163.89737672330656,62.62171370749238,179.56798525398935,179.80535183884012,30.0,32.11571708368048,156.34484056441445,37.784068080962264,177.5140670242546,176.5104064667321,140.59863065132043,141.18777805570252,145.97405030126149,151.29228373401608,129.87013695902078,46.98828162634734,95.16328465437428]"
 "cf_photosic_reference_as_tuple,20",0.082843553147,"[84.60641384124756,93.32310140132904,160.6474506855011,118.69610130786896,149.33961510658264,120.1955097913742,143.7487918138504,121.4040058851242,141.8354469537735,120.70926904678345,142.6336634159088,119.34617042541504,145.00528156757355,117.46501207351685,144.83027637004852,122.64509081840515,133.309428691864,130.55277943611145,91.41407936811447,74.78745460510254]"