facebookresearch · teytaud · Nov 19, 2024 · Sep 12, 2024 · Oct 12, 2024 · Oct 12, 2024
diff --git a/nevergrad/benchmark/experiments.py b/nevergrad/benchmark/experiments.py
@@ -54,37 +54,6 @@
 #    list_optims = ["QOTPDE", "LQOTPDE", "LQODE"]
 #    list_optims = ["SPQODE", "SQOPSO", "DiagonalCMA"]
 def refactor_optims(x: tp.List[tp.Any]) -> tp.List[tp.Any]:  # type: ignore
-    #     return list(
-    #         np.random.choice(
-    #             [
-    #                 "RandomSearch",
-    #                 "AXP",
-    #                 "LognormalDiscreteOnePlusOne",
-    #                 "AX",
-    #                 "CMA",
-    #                 "Cobyla",
-    #                 "PCABO",
-    #                 "SMAC3",
-    #                 "NgIohTuned",
-    #                 "NGDSRW",
-    #                 "PymooBIPOP",
-    #                 "CMA",
-    #                 "PSO",
-    #                 "SQOPSO",
-    #                 "DE",
-    #                 "DiscreteLenglerOnePlusOne",
-    #                 "DiscreteOnePlusOne",
-    #                 "OnePlusOne",
-    #                 "DSproba",
-    #                 "MetaModel",
-    #             ],
-    #             1,
-    #         )
-    #     )
-    #     return [
-    #         np.random.choice(["RandomSearch", "AX", "CMA", "Cobyla", "PCABO", "SMAC3", "NgIohTuned", "NGDSRW"])
-    #     ]
-    # return ["NgIoh4", "NgIoh21", "NgIohTuned", "NgIohLn", "NgIohRS", "LBFGSB"]
     if False:  # np.random.randn() < 0.0:
         return list(
             np.random.choice(

diff --git a/nevergrad/benchmark/gymexperiments.py b/nevergrad/benchmark/gymexperiments.py
@@ -3,6 +3,7 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
+# import numpy as np
 import os
 import typing as tp
 from nevergrad.functions import gym as nevergrad_gym
@@ -22,86 +23,7 @@ def gym_problem_modifier(specific_problem):
 
 
 def gym_optimizer_modifier(optims):
-    #     return list(
-    #         np.random.choice(
-    #             [
-    #                 "RandomSearch",
-    #                 "AXP",
-    #                 "LognormalDiscreteOnePlusOne",
-    #                 "AX",
-    #                 "CMA",
-    #                 "Cobyla",
-    #                 "PCABO",
-    #                 "SMAC3",
-    #                 "NgIohTuned",
-    #                 "NGDSRW",
-    #                 "PymooBIPOP",
-    #                 "CMA",
-    #                 "PSO",
-    #                 "SQOPSO",
-    #                 "DE",
-    #                 "DiscreteLenglerOnePlusOne",
-    #                 "DiscreteOnePlusOne",
-    #                 "OnePlusOne",
-    #                 "DSproba",
-    #                 "MetaModel",
-    #             ],
-    #             1,
-    #         )
-    #     )
-    #     return list(
-    #         np.random.choice(
-    #             [
-    #                 "RandomSearch",
-    #                 "AX",
-    #                 "CMA",
-    #                 "Cobyla",
-    #                 "PCABO",
-    #                 "SMAC3",
-    #                 "NgIohTuned",
-    #                 "NGDSRW",
-    #                 "PymooBIPOP",
-    #                 "CMA",
-    #                 "PSO",
-    #                 "SQOPSO",
-    #                 "DE",
-    #                 "DiscreteLenglerOnePlusOne",
-    #                 "DiscreteOnePlusOne",
-    #                 "OnePlusOne",
-    #                 "DSproba",
-    #                 "MetaModel",
-    #             ],
-    #             3,
-    #         )
-    #     )
-    #     return list(
-    #         np.random.choice(
-    #             [
-    #                 "RandomSearch",
-    #                 "AXP",
-    #                 "LognormalDiscreteOnePlusOne",
-    #                 "AX",
-    #                 "CMA",
-    #                 "Cobyla",
-    #                 "PCABO",
-    #                 "SMAC3",
-    #                 "NgIohTuned",
-    #                 "NGDSRW",
-    #                 "PymooBIPOP",
-    #                 "CMA",
-    #                 "PSO",
-    #                 "SQOPSO",
-    #                 "DE",
-    #                 "DiscreteLenglerOnePlusOne",
-    #                 "DiscreteOnePlusOne",
-    #                 "OnePlusOne",
-    #                 "DSproba",
-    #                 "MetaModel",
-    #             ],
-    #             1,
-    #         )
-    #     )
-    print(optims)
+
     if os.environ.get("GYM_OPTIMIZER") is not None:
         optimizer_string = os.environ.get("GYM_OPTIMIZER")
         print(f"Considering optimizers with {optimizer_string} in their name.")

diff --git a/nevergrad/benchmark/plotting.py b/nevergrad/benchmark/plotting.py
@@ -402,6 +402,12 @@ def create_plots(
             name = "fight_all.png" if name == "fight_.png" else name
             name = compactize(name)
             fullname = name
+            if name == "fight_all.png":
+                with open(str(output_folder / name) + ".cp.txt", "w") as f:
+                    f.write(fullname)
+                    f.write("ranking:\n")
+                    for i, algo in enumerate(data_df.columns[:158]):
+                        f.write(f"  algo {i}: {algo}\n")
 
             if len(name) > 240:
                 hashcode = hashlib.md5(bytes(name, "utf8")).hexdigest()
@@ -415,13 +421,8 @@ def create_plots(
             )
             fplotter = FightPlotter(data_df)
             if name == "fight_all.png":
-                with open(str(output_folder / name) + ".cp.txt", "w") as f:
-                    f.write(fullname)
-                    f.write("ranking:\n")
-                    global pure_algorithms
-                    pure_algorithms = list(data_df.columns[:])
-                    for i, algo in enumerate(data_df.columns[:158]):
-                        f.write(f"  algo {i}: {algo}\n")
+                global pure_algorithms
+                pure_algorithms = list(data_df.columns[:])
             if name == "fight_all.png":
                 fplotter.save(str(output_folder / "fight_all_pure.png"), dpi=_DPI)
             else:
@@ -445,9 +446,14 @@ def create_plots(
     out_filepath = output_folder / "xpresults_all.png"
     try:
         data = XpPlotter.make_data(df, normalized_loss=True)
-        xpplotter = XpPlotter(
-            data, title=os.path.basename(output_folder), name_style=name_style, xaxis=xpaxis, pure_only=True
-        )
+        for pure_only in [False, True]:
+            xpplotter = XpPlotter(
+                data,
+                title=os.path.basename(output_folder),
+                name_style=name_style,
+                xaxis=xpaxis,
+                pure_only=pure_only,
+            )
     except Exception as e:
         lower = 0
         upper = len(df)
@@ -622,11 +628,11 @@ def __init__(
             sorted_optimizers = [
                 o for o in sorted_optimizers if o + " " in [p[: (len(o) + 1)] for p in pure_algorithms]
             ]
-            with open("rnk__" + str(title) + ".cp.txt", "w") as f:
-                f.write(compactize(title))
-                f.write("ranking:\n")
-                for i, algo in reversed(list(enumerate(sorted_optimizers))):
-                    f.write(f"  algo {i}: {algo} (x)\n")
+        with open(("rnk__" if not pure_only else "rnkpure__") + str(title) + ".cp.txt", "w") as f:
+            f.write(compactize(title))
+            f.write("ranking:\n")
+            for i, algo in reversed(list(enumerate(sorted_optimizers))):
+                f.write(f"  algo {i}: {algo} (x)\n")
             # print(sorted_optimizers, " merged with ", pure_algorithms)
             # print("Leads to ", sorted_optimizers)
         self._fig = plt.figure()

diff --git a/nevergrad/benchmark/xpbase.py b/nevergrad/benchmark/xpbase.py
@@ -291,6 +291,11 @@ def _run_with_error(self, callbacks: tp.Optional[tp.Dict[str, obase._OptimCallBa
             try:
                 # call the actual Optimizer.minimize method because overloaded versions could alter the worklflow
                 # and provide unfair comparisons  (especially for parallelized settings)
+                # import signal
+                # def handler(signum, frame):
+                #    raise Exception("Time exceeded!")
+                # signal.signal(signal.SIGALRM, handler)
+                # signal.alarm(int(2.8 * 3600 * 24))
                 if self.suggestions is not None:
                     for s in self.suggestions:
                         self._optimizer.suggest(s)
@@ -302,6 +307,7 @@ def _run_with_error(self, callbacks: tp.Optional[tp.Dict[str, obase._OptimCallBa
                     constraint_violation=self.constraint_violation,
                     max_time=3600 * 24 * 2.5,
                 )
+                # signal.alarm(0)
             except Exception as e:  # pylint: disable=broad-except
                 self._log_results(pfunc, t0, self._optimizer.num_ask)
                 raise e

diff --git a/nevergrad/optimization/optimizerlib.py b/nevergrad/optimization/optimizerlib.py
@@ -1173,10 +1173,7 @@ def __init__(
             objectives={"result": ObjectiveProperties(minimize=True)},
             #                            parameter_constraints=["x + y <= 2.0"],  # Optional.
         )
-
-    # def _internal_provide_recommendation(
-    #    self,
-    # ) -> tp.ArrayLike:  # This is NOT the naive version. We deal with noise.
+        self._trials = []  # type: ignore
 
     def _internal_ask_candidate(self) -> p.Parameter:
         def invsig(x):
@@ -1185,11 +1182,14 @@ def p(x):
 
             return np.log(p(x) / (1 - p(x)))
 
-        trial_index_to_param, _ = self.ax_client.get_next_trials(max_trials=1)
-        assert len(trial_index_to_param) == 1
+        if len(self._trials) == 0:
+            trial_index_to_param, _ = self.ax_client.get_next_trials(max_trials=1)
+            for _, _trial in trial_index_to_param.items():
+                trial = _trial
+                self._trials += [trial]
+        trial = self._trials[0]
+        self._trials = self._trials[1:]
         vals = np.zeros(self.dimension)
-        for _, _trial in trial_index_to_param.items():
-            trial = _trial
         for i in range(self.dimension):
             vals[i] = invsig(trial["x" + str(i)])
         candidate = self.parametrization.spawn_child().set_standardized_data(vals)
@@ -3969,6 +3969,13 @@ def _select_optimizer_cls(self) -> base.OptCls:
         return optCls
 
 
+# @registry.register
+# class MetaBO(NGOptBase):
+#    """Nevergrad optimizer by competence map. You might modify this one for designing your own competence map."""
+#
+#    def _select_optimizer_cls(self) -> base.OptCls:
+#        optCls: base.OptCls = NGOptBase
+#        funcinfo = self.parametrization.function
 @registry.register
 class NGO(NGOptBase):  # compatibility
     pass

diff --git a/scripts/dagstuhloid.sh b/scripts/dagstuhloid.sh
@@ -6,10 +6,13 @@
 #SBATCH --partition=scavenge
 #SBATCH --nodes=1
 #SBATCH --cpus-per-task=70
-#SBATCH -a 0-323
+#SBATCH -a 0-76
 
 
-tasks=(nozp_noms_bbob multi_structured_ng_full_gym lsgo smallbudget_lsgo ng_gym sequential_fastgames small_deterministic_gym_multi tiny_deterministic_gym_multi zp_pbbob zp_ms_bbob aquacrop_fao bonnans deceptive double_o_seven fishing instrum_discrete keras_tuning mldakmeans mltuning mono_rocket multimodal multiobjective_example multiobjective_example_hd multiobjective_example_many_hd naive_seq_keras_tuning naive_seq_mltuning naive_veryseq_keras_tuning naivemltuning nano_naive_seq_mltuning nano_naive_veryseq_mltuning nano_seq_mltuning nano_veryseq_mltuning neuro_oneshot_mltuning pbbob pbo_reduced_suite reduced_yahdlbbbob rocket seq_keras_tuning seq_mltuning sequential_instrum_discrete sequential_topology_optimization spsa_benchmark topology_optimization ultrasmall_photonics ultrasmall_photonics2 veryseq_keras_tuning yabbob yabigbbob yaboundedbbob yaboxbbob yahdbbob yahdnoisybbob yamegapenbbob yamegapenboundedbbob yamegapenboxbbob yanoisybbob yaonepenbbob yaonepenboundedbbob yaonepenboxbbob yaonepennoisybbob yaonepenparabbob yaonepensmallbbob yaparabbob yapenbbob yapenboundedbbob yapenboxbbob yapennoisybbob yapenparabbob yapensmallbbob yasmallbbob yatinybbob yatuningbbob ms_bbob ranknoisy powersystems verysmall_photonics verysmall_photonics2)
+tasks=(nozp_noms_bbob multi_structured_ng_full_gym lsgo smallbudget_lsgo ng_gym sequential_fastgames small_deterministic_gym_multi tiny_deterministic_gym_multi zp_pbbob zp_ms_bbob aquacrop_fao bonnans deceptive double_o_seven fishing instrum_discrete keras_tuning mldakmeans mltuning mono_rocket multimodal multiobjective_example multiobjective_example_hd multiobjective_example_many_hd naive_seq_keras_tuning naive_seq_mltuning naive_veryseq_keras_tuning naivemltuning nano_naive_seq_mltuning nano_naive_veryseq_mltuning nano_seq_mltuning nano_veryseq_mltuning neuro_oneshot_mltuning pbbob pbo_reduced_suite reduced_yahdlbbbob rocket seq_keras_tuning seq_mltuning sequential_instrum_discrete sequential_topology_optimization spsa_benchmark topology_optimization ultrasmall_photonics ultrasmall_photonics2 veryseq_keras_tuning yahdlbbbob yabbob yabigbbob yaboundedbbob yaboxbbob yahdbbob yahdnoisybbob yamegapenbbob yamegapenboundedbbob yamegapenboxbbob yanoisybbob yaonepenbbob yaonepenboundedbbob yaonepenboxbbob yaonepennoisybbob yaonepenparabbob yaonepensmallbbob yaparabbob yapenbbob yapenboundedbbob yapenboxbbob yapennoisybbob yapenparabbob yapensmallbbob yasmallbbob yatinybbob yatuningbbob ms_bbob ranknoisy powersystems verysmall_photonics verysmall_photonics2)
+#tasks=(keras_tuning  lsgo  mono_rocket  naive_seq_keras_tuning  naive_seq_mltuning  naive_veryseq_keras_tuning  nano_naive_seq_mltuning  nano_seq_mltuning  ranknoisy  rocket  seq_keras_tuning  seq_mltuning  veryseq_keras_tuning  yahdnoisybbob  yaonepennoisybbob  yapennoisybbob  )
+# TODO remove above
+
 #tasks=(ng_gym small_deterministic_gym_multi tiny_deterministic_gym_multi zp_pbbob zp_ms_bbob aquacrop_fao bonnans deceptive double_o_seven fishing instrum_discrete keras_tuning mldakmeans mltuning mono_rocket multimodal multiobjective_example multiobjective_example_hd multiobjective_example_many_hd naive_seq_keras_tuning naive_seq_mltuning naive_veryseq_keras_tuning naivemltuning nano_naive_seq_mltuning nano_naive_veryseq_mltuning nano_seq_mltuning nano_veryseq_mltuning neuro_oneshot_mltuning pbbob pbo_reduced_suite reduced_yahdlbbbob rocket seq_keras_tuning seq_mltuning sequential_instrum_discrete sequential_topology_optimization spsa_benchmark topology_optimization ultrasmall_photonics ultrasmall_photonics2 veryseq_keras_tuning yabbob yabigbbob yaboundedbbob yaboxbbob yahdbbob yahdnoisybbob yamegapenbbob yamegapenboundedbbob yamegapenboxbbob yanoisybbob yaonepenbbob yaonepenboundedbbob yaonepenboxbbob yaonepennoisybbob yaonepenparabbob yaonepensmallbbob yaparabbob yapenbbob yapenboundedbbob yapenboxbbob yapennoisybbob yapenparabbob yapensmallbbob yasmallbbob yatinybbob yatuningbbob ms_bbob ranknoisy powersystems verysmall_photonics verysmall_photonics2)
 task=${tasks[SLURM_ARRAY_TASK_ID % ${#tasks[@]} ]}
 
@@ -19,6 +22,7 @@ task=${tasks[SLURM_ARRAY_TASK_ID % ${#tasks[@]} ]}
 #
 #conda info
 
+
 #echo Starting at
 #date
 # num_workers is the number of processes. Maybe use a bit more than the number of cores at the line "cpus-per-task"

diff --git a/scripts/get_statistics.sh b/scripts/get_statistics.sh
@@ -3,9 +3,9 @@
 pbs=`ls rnk*.txt | sed 's/rnk__//g' | sed 's/_plots.*//g'`
 
 (
-echo '\begin{tabular}{|c|c|c|c|}'
+echo '\begin{tabular}{|p{2cm}|c|c|c|}'
 echo '\hline'
-echo 'Best algorithm & Dimension & Budget & Problem name \\'
+echo 'Best algorithm & Problem name & Dimension & Budget \\'
 echo '\hline'
 
 for pb in $pbs

diff --git a/scripts/multirank.sh b/scripts/multirank.sh
@@ -1,5 +1,9 @@
 #!/bin/bash
 
+echo DEPRECATED
+
+exit 1
+
 (
 echo 'import matplotlib'
 echo 'import matplotlib.pyplot as plt'
@@ -22,28 +26,29 @@ echo 'plt.clf()'
 echo  "labels = []"
 echo  "numbers = []"
 echo "# $num best"
-for a in RandomSearch AX AXP Cobyla PCABO SMAC3 NgIohTuned NGDSRW PymooBIPOP CMA PSO SQOPSO DE DiscreteLenglerOnePlusOne DiscreteOnePlusOne OnePlusOne DSproba MetaModel LognormalDiscreteOnePlusOne 
+for a in RandomSearch AXP Cobyla PCABO SMAC3 NgIohTuned PymooBIPOP CMA PSO SQOPSO DE DiscreteLenglerOnePlusOne DiscreteOnePlusOne OnePlusOne DSproba MetaModel LognormalDiscreteOnePlusOne 
+
 do
 echo -n "# $a is in the $num best in this number of problems:"
 number=$( 
 for k in rnk*.txt
 do
-if [ $ag="yes" ]; then
-    grep ':' $k | sed 's/.*://g' | sed 's/ (.*//g' | egrep '[a-zA-Z0-9]' | egrep -v ' NgIohTuned | NGDSRW | AX | AXP | LognormalDiscreteOnePlusOne | PCABO | PSO | PymooBIPOP | CMA ' | head -n $num | grep "^ $a$"
+if [ "$ag" == yes ]; then
+    grep ':' $k | grep -v ' AX ' | sed 's/.*://g' | sed 's/ (.*//g' | egrep '[a-zA-Z0-9]' | egrep -v ' NgIohTuned | NGDSRW | AX | SMAC3 | LognormalDiscreteOnePlusOne | PCABO | PSO | PymooBIPOP | CMA ' | head -n $num | grep "^ $a$"
 else
-    grep ':' $k | sed 's/.*://g' | sed 's/ (.*//g' | egrep '[a-zA-Z0-9]' | head -n $num | grep "^ $a$"
+    grep ':' $k | grep -v ' AX ' | grep -v ' NGDSRW ' | sed 's/.*://g' | sed 's/ (.*//g' | egrep '[a-zA-Z0-9]' | head -n $num | grep "^ $a$"
 fi
 done | wc -l )
 echo $number
-if [ $ag="yes" ]; then
-echo "labels += [\" $a \" + '(' + str( $number ) + ')' ];numbers += [ $number + .2 ]" | egrep -v ' NgIohTuned | NGDSRW | AX | AXP | PCABO | PSO | PymooBIPOP | CMA | LognormalDiscreteOnePlusOne '
+if [ "$ag" == yes ]; then
+echo "labels += [\" $a \" + '(' + str( $number ) + ')' ];numbers += [ $number + .2 ]" | egrep -v ' NgIohTuned | NGDSRW | AX | SMAC3 | PCABO | PSO | PymooBIPOP | CMA | LognormalDiscreteOnePlusOne '
 else
 echo "labels += [\" $a \" + '(' + str( $number ) + ')' ];numbers += [ $number + .2 ]"
 fi
 done
 echo "plt.pie(numbers, labels=labels)"
 echo "plt.title(title)"
-if [ $ag="yes" ]; then
+if [ "$ag" == yes ]; then
 echo "plt.savefig(f'agpie{num}.png')"
 else
 echo "plt.savefig(f'pie{num}.png')"