From e9a9aabf147d6f44c1fcc803500d349d294e4805 Mon Sep 17 00:00:00 2001 From: Olivier Teytaud Date: Thu, 12 Sep 2024 14:00:34 +0200 Subject: [PATCH 1/8] timelimit --- nevergrad/benchmark/xpbase.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nevergrad/benchmark/xpbase.py b/nevergrad/benchmark/xpbase.py index 4011c66e4..a1b416f3b 100644 --- a/nevergrad/benchmark/xpbase.py +++ b/nevergrad/benchmark/xpbase.py @@ -12,6 +12,7 @@ import nevergrad.common.typing as ngtp import typing as tp import numpy as np +from nevergrad.optimization import callbacks from nevergrad.parametrization import parameter as p from nevergrad.common import decorators from nevergrad.common import errors @@ -289,6 +290,7 @@ def _run_with_error(self, callbacks: tp.Optional[tp.Dict[str, obase._OptimCallBa try: # call the actual Optimizer.minimize method because overloaded versions could alter the worklflow # and provide unfair comparisons (especially for parallelized settings) + optimizer.register_callback("ask", callbacks.EarlyStopping.timer(3600*(48+24))) obase.Optimizer.minimize( self._optimizer, pfunc, From c70b4226e8fbac517a5775e5797818189ba1a6bd Mon Sep 17 00:00:00 2001 From: Olivier Teytaud Date: Sat, 12 Oct 2024 15:08:08 +0200 Subject: [PATCH 2/8] timelimit --- nevergrad/benchmark/experiments.py | 30 ++++++++++ nevergrad/benchmark/gymexperiments.py | 79 ++++++++++++++++++++++++++ nevergrad/benchmark/xpbase.py | 5 +- nevergrad/optimization/base.py | 7 ++- nevergrad/optimization/optimizerlib.py | 66 ++++++++++++++++++++- nevergrad/optimization/recastlib.py | 4 +- scripts/dagstuhloid.sh | 34 +++++------ scripts/get_statistics.sh | 21 +++++++ scripts/latexize.sh | 11 ++-- scripts/multirank.sh | 62 ++++++++++++++++++++ scripts/plot_dagstuhloid.sh | 9 ++- scripts/tex/end.tex | 15 +++++ 12 files changed, 315 insertions(+), 28 deletions(-) create mode 100755 scripts/get_statistics.sh create mode 100755 scripts/multirank.sh diff --git a/nevergrad/benchmark/experiments.py b/nevergrad/benchmark/experiments.py index a94fc996f..ec17444fb 100644 --- a/nevergrad/benchmark/experiments.py +++ b/nevergrad/benchmark/experiments.py @@ -54,6 +54,36 @@ # list_optims = ["QOTPDE", "LQOTPDE", "LQODE"] # list_optims = ["SPQODE", "SQOPSO", "DiagonalCMA"] def refactor_optims(x: tp.List[tp.Any]) -> tp.List[tp.Any]: # type: ignore + return list( + np.random.choice( + [ + "RandomSearch", + "AXP", + "LognormalDiscreteOnePlusOne", + "AX", + "CMA", + "Cobyla", + "PCABO", + "SMAC3", + "NgIohTuned", + "NGDSRW", + "PymooBIPOP", + "CMA", + "PSO", + "SQOPSO", + "DE", + "DiscreteLenglerOnePlusOne", + "DiscreteOnePlusOne", + "OnePlusOne", + "DSproba", + "MetaModel", + ], + 1, + ) + ) + return [ + np.random.choice(["RandomSearch", "AX", "CMA", "Cobyla", "PCABO", "SMAC3", "NgIohTuned", "NGDSRW"]) + ] # return ["NgIoh4", "NgIoh21", "NgIohTuned", "NgIohLn", "NgIohRS", "LBFGSB"] if False: # np.random.randn() < 0.0: return list( diff --git a/nevergrad/benchmark/gymexperiments.py b/nevergrad/benchmark/gymexperiments.py index 82c2dc4e2..f8f8ba2e9 100644 --- a/nevergrad/benchmark/gymexperiments.py +++ b/nevergrad/benchmark/gymexperiments.py @@ -22,6 +22,85 @@ def gym_problem_modifier(specific_problem): def gym_optimizer_modifier(optims): + return list( + np.random.choice( + [ + "RandomSearch", + "AXP", + "LognormalDiscreteOnePlusOne", + "AX", + "CMA", + "Cobyla", + "PCABO", + "SMAC3", + "NgIohTuned", + "NGDSRW", + "PymooBIPOP", + "CMA", + "PSO", + "SQOPSO", + "DE", + "DiscreteLenglerOnePlusOne", + "DiscreteOnePlusOne", + "OnePlusOne", + "DSproba", + "MetaModel", + ], + 1, + ) + ) + return list( + np.random.choice( + [ + "RandomSearch", + "AX", + "CMA", + "Cobyla", + "PCABO", + "SMAC3", + "NgIohTuned", + "NGDSRW", + "PymooBIPOP", + "CMA", + "PSO", + "SQOPSO", + "DE", + "DiscreteLenglerOnePlusOne", + "DiscreteOnePlusOne", + "OnePlusOne", + "DSproba", + "MetaModel", + ], + 3, + ) + ) + return list( + np.random.choice( + [ + "RandomSearch", + "AXP", + "LognormalDiscreteOnePlusOne", + "AX", + "CMA", + "Cobyla", + "PCABO", + "SMAC3", + "NgIohTuned", + "NGDSRW", + "PymooBIPOP", + "CMA", + "PSO", + "SQOPSO", + "DE", + "DiscreteLenglerOnePlusOne", + "DiscreteOnePlusOne", + "OnePlusOne", + "DSproba", + "MetaModel", + ], + 1, + ) + ) print(optims) if os.environ.get("GYM_OPTIMIZER") is not None: optimizer_string = os.environ.get("GYM_OPTIMIZER") diff --git a/nevergrad/benchmark/xpbase.py b/nevergrad/benchmark/xpbase.py index a1b416f3b..4254d5874 100644 --- a/nevergrad/benchmark/xpbase.py +++ b/nevergrad/benchmark/xpbase.py @@ -12,7 +12,7 @@ import nevergrad.common.typing as ngtp import typing as tp import numpy as np -from nevergrad.optimization import callbacks +from nevergrad.optimization import callbacks as ngcallbacks from nevergrad.parametrization import parameter as p from nevergrad.common import decorators from nevergrad.common import errors @@ -290,13 +290,14 @@ def _run_with_error(self, callbacks: tp.Optional[tp.Dict[str, obase._OptimCallBa try: # call the actual Optimizer.minimize method because overloaded versions could alter the worklflow # and provide unfair comparisons (especially for parallelized settings) - optimizer.register_callback("ask", callbacks.EarlyStopping.timer(3600*(48+24))) + # self._optimizer.register_callback("ask", ngcallbacks.EarlyStopping.timer(3600*(48+24))) obase.Optimizer.minimize( self._optimizer, pfunc, batch_mode=executor.batch_mode, executor=executor, constraint_violation=self.constraint_violation, + max_time=3600 * 24 * 2.5, ) except Exception as e: # pylint: disable=broad-except self._log_results(pfunc, t0, self._optimizer.num_ask) diff --git a/nevergrad/optimization/base.py b/nevergrad/optimization/base.py index 0bb6bd0f0..a9a21fea1 100644 --- a/nevergrad/optimization/base.py +++ b/nevergrad/optimization/base.py @@ -3,6 +3,7 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. +import time import pickle import warnings from pathlib import Path @@ -615,6 +616,7 @@ def minimize( batch_mode: bool = False, verbosity: int = 0, constraint_violation: tp.Any = None, + max_time: tp.Optional[float] = None, ) -> p.Parameter: """Optimization (minimization) procedure @@ -664,7 +666,10 @@ def minimize( remaining_budget = self.budget - self.num_ask first_iteration = True # - while remaining_budget or self._running_jobs or self._finished_jobs: + t0 = time.time() + while (remaining_budget or self._running_jobs or self._finished_jobs) and ( + max_time is None or time.time() < t0 + max_time + ): # # # # # Update optimizer with finished jobs # # # # # # this is the first thing to do when resuming an existing optimization run # process finished diff --git a/nevergrad/optimization/optimizerlib.py b/nevergrad/optimization/optimizerlib.py index 127a52234..4ba2857d9 100644 --- a/nevergrad/optimization/optimizerlib.py +++ b/nevergrad/optimization/optimizerlib.py @@ -1133,6 +1133,70 @@ def _internal_tell_not_asked(self, candidate: p.Parameter, loss: tp.FloatLoss) - raise errors.TellNotAskedNotSupportedError +# pylint: disable=too-many-instance-attributes + + +@registry.register +class AXP(base.Optimizer): + """Test-based population-size adaptation. + + Population-size equal to lambda = 4 x dimension. + Test by comparing the first fifth and the last fifth of the 5lambda evaluations. + + Caution + ------- + This optimizer is probably wrong. + """ + + def __init__( + self, parametrization: IntOrParameter, budget: tp.Optional[int] = None, num_workers: int = 1 + ) -> None: + super().__init__(parametrization, budget=budget, num_workers=num_workers) + try: + from ax.service.ax_client import AxClient, ObjectiveProperties + except Exception as e: + print(f"Pb for creating AX solver") + raise e + self.ax_parametrization = [ + {"name": "x" + str(i), "type": "range", "bounds": [0.0, 1.0]} for i in range(self.dimension) + ] + self.ax_client = AxClient() + self.ax_client.create_experiment( + name="ax_optimization", + parameters=self.ax_parametrization, + objectives={"result": ObjectiveProperties(minimize=True)}, + # parameter_constraints=["x + y <= 2.0"], # Optional. + ) + + # def _internal_provide_recommendation( + # self, + # ) -> tp.ArrayLike: # This is NOT the naive version. We deal with noise. + + def _internal_ask_candidate(self) -> p.Parameter: + def invsig(x): + def p(x): + return np.clip(x, 1e-15, 1.0 - 1e-15) + + return np.log(p(x) / (1 - p(x))) + + trial_index_to_param, _ = self.ax_client.get_next_trials(max_trials=1) + assert len(trial_index_to_param) == 1 + vals = np.zeros(self.dimension) + for a, _trial in trial_index_to_param.items(): + trial = _trial + for i in range(self.dimension): + vals[i] = invsig(trial["x" + str(i)]) + candidate = self.parametrization.spawn_child().set_standardized_data(vals) + candidate._meta["trial_index"] = self.num_ask + return candidate + + def _internal_tell_candidate(self, candidate: p.Parameter, loss: tp.FloatLoss) -> None: + self.ax_client.complete_trial(trial_index=candidate._meta["trial_index"], raw_data=loss) + + def _internal_tell_not_asked(self, candidate: p.Parameter, loss: tp.FloatLoss) -> None: + raise errors.TellNotAskedNotSupportedError + + class PCEDA(EDA): _POPSIZE_ADAPTATION = True _COVARIANCE_MEMORY = False @@ -3078,7 +3142,7 @@ def __init__( obj_fun=None, # Assuming that this is not used :-) DoE_size=init_budget if init_budget is not None else 5, max_FEs=budget, - verbose=True, + verbose=False, n_point=1, # We start with a sequential procedure, maybe we'll extend in a second moment n_components=cfg.n_components, acquisition_optimization={"optimizer": "BFGS"}, diff --git a/nevergrad/optimization/recastlib.py b/nevergrad/optimization/recastlib.py index 3b5c5bb3b..efedefa44 100644 --- a/nevergrad/optimization/recastlib.py +++ b/nevergrad/optimization/recastlib.py @@ -314,7 +314,9 @@ def smac2_obj(p, seed: int = 0): return res # scenario = Scenario({'cs': cs, 'run_obj': smac2_obj, 'runcount-limit': remaining, 'deterministic': True}) - scenario = Scenario(cs, deterministic=True, n_trials=int(remaining)) + scenario = Scenario( + cs, deterministic=True, n_trials=int(remaining) + ) # , output_directory="/dev/null") smac = HyperparameterOptimizationFacade(scenario, smac2_obj) res = smac.optimize() diff --git a/scripts/dagstuhloid.sh b/scripts/dagstuhloid.sh index 8381c6453..848b13364 100755 --- a/scripts/dagstuhloid.sh +++ b/scripts/dagstuhloid.sh @@ -6,31 +6,31 @@ #SBATCH --partition=scavenge #SBATCH --nodes=1 #SBATCH --cpus-per-task=70 -#SBATCH -a 0-73 +#SBATCH -a 0-323 tasks=(nozp_noms_bbob multi_structured_ng_full_gym lsgo smallbudget_lsgo ng_gym sequential_fastgames small_deterministic_gym_multi tiny_deterministic_gym_multi zp_pbbob zp_ms_bbob aquacrop_fao bonnans deceptive double_o_seven fishing instrum_discrete keras_tuning mldakmeans mltuning mono_rocket multimodal multiobjective_example multiobjective_example_hd multiobjective_example_many_hd naive_seq_keras_tuning naive_seq_mltuning naive_veryseq_keras_tuning naivemltuning nano_naive_seq_mltuning nano_naive_veryseq_mltuning nano_seq_mltuning nano_veryseq_mltuning neuro_oneshot_mltuning pbbob pbo_reduced_suite reduced_yahdlbbbob rocket seq_keras_tuning seq_mltuning sequential_instrum_discrete sequential_topology_optimization spsa_benchmark topology_optimization ultrasmall_photonics ultrasmall_photonics2 veryseq_keras_tuning yabbob yabigbbob yaboundedbbob yaboxbbob yahdbbob yahdnoisybbob yamegapenbbob yamegapenboundedbbob yamegapenboxbbob yanoisybbob yaonepenbbob yaonepenboundedbbob yaonepenboxbbob yaonepennoisybbob yaonepenparabbob yaonepensmallbbob yaparabbob yapenbbob yapenboundedbbob yapenboxbbob yapennoisybbob yapenparabbob yapensmallbbob yasmallbbob yatinybbob yatuningbbob ms_bbob ranknoisy powersystems verysmall_photonics verysmall_photonics2) #tasks=(ng_gym small_deterministic_gym_multi tiny_deterministic_gym_multi zp_pbbob zp_ms_bbob aquacrop_fao bonnans deceptive double_o_seven fishing instrum_discrete keras_tuning mldakmeans mltuning mono_rocket multimodal multiobjective_example multiobjective_example_hd multiobjective_example_many_hd naive_seq_keras_tuning naive_seq_mltuning naive_veryseq_keras_tuning naivemltuning nano_naive_seq_mltuning nano_naive_veryseq_mltuning nano_seq_mltuning nano_veryseq_mltuning neuro_oneshot_mltuning pbbob pbo_reduced_suite reduced_yahdlbbbob rocket seq_keras_tuning seq_mltuning sequential_instrum_discrete sequential_topology_optimization spsa_benchmark topology_optimization ultrasmall_photonics ultrasmall_photonics2 veryseq_keras_tuning yabbob yabigbbob yaboundedbbob yaboxbbob yahdbbob yahdnoisybbob yamegapenbbob yamegapenboundedbbob yamegapenboxbbob yanoisybbob yaonepenbbob yaonepenboundedbbob yaonepenboxbbob yaonepennoisybbob yaonepenparabbob yaonepensmallbbob yaparabbob yapenbbob yapenboundedbbob yapenboxbbob yapennoisybbob yapenparabbob yapensmallbbob yasmallbbob yatinybbob yatuningbbob ms_bbob ranknoisy powersystems verysmall_photonics verysmall_photonics2) -task=${tasks[SLURM_ARRAY_TASK_ID]} +task=${tasks[SLURM_ARRAY_TASK_ID % ${#tasks[@]} ]} -echo task attribution $SLURM_ARRAY_TASK_ID $task -echo Keras/TF versions: -pip show keras tensorflow tensorflow-estimator +#echo task attribution $SLURM_ARRAY_TASK_ID $task +#echo Keras/TF versions: +#pip show keras tensorflow tensorflow-estimator +# +#conda info -conda info - -echo Starting at -date +#echo Starting at +#date # num_workers is the number of processes. Maybe use a bit more than the number of cores at the line "cpus-per-task" # above. -python -m nevergrad.benchmark $task --num_workers=73 2>&1 | tail -n 50 -python -m nevergrad.benchmark $task --num_workers=73 2>&1 | tail -n 50 -python -m nevergrad.benchmark $task --num_workers=73 2>&1 | tail -n 50 -python -m nevergrad.benchmark $task --num_workers=73 2>&1 | tail -n 50 -python -m nevergrad.benchmark $task --num_workers=73 2>&1 | tail -n 50 -python -m nevergrad.benchmark $task --num_workers=73 2>&1 | tail -n 50 -python -m nevergrad.benchmark $task --num_workers=73 2>&1 | tail -n 50 -python -m nevergrad.benchmark $task --num_workers=73 2>&1 | tail -n 50 +python -m nevergrad.benchmark $task --num_workers=73 2>&1 | tail -n 12 +#python -m nevergrad.benchmark $task --num_workers=73 2>&1 | tail -n 50 +#python -m nevergrad.benchmark $task --num_workers=73 2>&1 | tail -n 50 +#python -m nevergrad.benchmark $task --num_workers=73 2>&1 | tail -n 50 +#python -m nevergrad.benchmark $task --num_workers=73 2>&1 | tail -n 50 +#python -m nevergrad.benchmark $task --num_workers=73 2>&1 | tail -n 50 +#python -m nevergrad.benchmark $task --num_workers=73 2>&1 | tail -n 50 +#python -m nevergrad.benchmark $task --num_workers=73 2>&1 | tail -n 50 echo task over $SLURM_ARRAY_TASK_ID $task echo Finishing at date diff --git a/scripts/get_statistics.sh b/scripts/get_statistics.sh new file mode 100755 index 000000000..04479796e --- /dev/null +++ b/scripts/get_statistics.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +pbs=`ls rnk*.txt | sed 's/rnk__//g' | sed 's/_plots.*//g'` + +( +echo '\begin{tabular}{|c|c|c|c|}' +echo '\hline' +echo 'Best algorithm & Dimension & Budget & Problem name \\' +echo '\hline' + +for pb in $pbs +do + +dim=$( ls -ctr ${pb}_plots/fig*dim*.png | grep -v ',' | grep pure | grep -v block | grep -v usefu | sed 's/.*dimension//g' | sed 's/.png.*//g' | awk '{ sum += $1 } END {print sum/NR}' ) + +budget=$( ls -ctr ${pb}_plots/fig*budg*.png | grep -v ',' | sed 's/.*budget//g' | grep pure | sed 's/.png.*//g' | awk '{ sum += $1 } END { print(sum/NR) }' ) +echo GO $pb $dim $budget $( grep algo rnk__${pb}_plots.cp.txt | grep ':' | head -n 1 | sed 's/.*: //g' | sed 's/ (.*//g' ) +done 2>&1 | grep '^GO' | sed 's/^GO //g' | grep ' .* .* ' | sort -k 4,4 | awk ' { if ( $4 != last ) { print "\\hline " ; print $4, "&", $1, "&", $2, "&", $3, "\\\\"; last=$4 } else {print "&", $1, "&", $2, "&", $3, "\\\\" } }' | sed 's/_/ /g' +echo '\hline' +echo '\end{tabular}' +) > agtable.tex diff --git a/scripts/latexize.sh b/scripts/latexize.sh index a9d2f7dfc..3d3aaf2bf 100755 --- a/scripts/latexize.sh +++ b/scripts/latexize.sh @@ -1,6 +1,6 @@ #!/bin/bash -pip install img2pdf +python -m pip install img2pdf allplots="" @@ -43,7 +43,8 @@ for v in zp_ms_bbob_plots/fight_translation_factor0.01.png_pure.png zp_ms_bbob_p do convert $v -trim +repage prout.png cp prout $v - img2pdf -o ${v}.pdf $v + # img2pdf -o ${v}.pdf $v + python -m img2pdf -o ${v}.pdf $v done for u in $allplots do @@ -57,9 +58,9 @@ timeout 10 cat scripts/txt/`echo $u | sed 's/_plots/.txt/g' | sed 's/\///g'` ( convert ${u}/fight_all_pure.png -trim +repage ${u}/fight_all_pure.pre.png -img2pdf -o ${u}/fight_all_pure.png.pdf ${u}/fight_all_pure.pre.png +python -m img2pdf -o ${u}/fight_all_pure.png.pdf ${u}/fight_all_pure.pre.png convert ${u}/xpresults_all.png -trim +repage ${u}/xpresults_all.pre.png -img2pdf -o ${u}/xpresults_all.png.pdf ${u}/xpresults_all.pre.png +python -m img2pdf -o ${u}/xpresults_all.png.pdf ${u}/xpresults_all.pre.png ) 2>&1 | cat > logconvert${uminus}.log ) & echo " " @@ -175,6 +176,8 @@ cat tmp.tex.tmp #ls `ls $v | sed 's/\.tex/\.pdf/g'` | sed 's/.*/\\includegraphics[width=.99\\textwidth]{{&}}\\\\/g' #done #done +echo '\section{Agregated table}' +cat agtable.tex cat scripts/tex/end.tex ) >> dagstuhloid.tex for v in competition.tex dagstuhloid.tex do diff --git a/scripts/multirank.sh b/scripts/multirank.sh new file mode 100755 index 000000000..dee57071d --- /dev/null +++ b/scripts/multirank.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +( +echo 'import matplotlib' +echo 'import matplotlib.pyplot as plt' + +#for k in rnk*.txt +#do +#echo -n "# $k :" +#grep ':' $k | sed 's/.*://g' | sed 's/ (.*//g' | egrep '[a-zA-Z0-9]' | sort | uniq | wc -l +# +#done + +for ag in yes no +do +for num in 1 2 4 8 +do +echo "num=$num" +echo "numt=$( ls rnk* | wc -l )" +echo "title=f'How many times in the {num} best ? out of {numt}'" +echo 'plt.clf()' +echo "labels = []" +echo "numbers = []" +echo "# $num best" +for a in RandomSearch AX AXP Cobyla PCABO SMAC3 NgIohTuned NGDSRW PymooBIPOP CMA PSO SQOPSO DE DiscreteLenglerOnePlusOne DiscreteOnePlusOne OnePlusOne DSproba MetaModel LognormalDiscreteOnePlusOne +do +echo -n "# $a is in the $num best in this number of problems:" +number=$( +for k in rnk*.txt +do +if [ $ag="yes" ]; then + grep ':' $k | sed 's/.*://g' | sed 's/ (.*//g' | egrep '[a-zA-Z0-9]' | egrep -v ' NgIohTuned | NGDSRW | AX | AXP | LognormalDiscreteOnePlusOne | PCABO | PSO | PymooBIPOP | CMA ' | head -n $num | grep "^ $a$" +else + grep ':' $k | sed 's/.*://g' | sed 's/ (.*//g' | egrep '[a-zA-Z0-9]' | head -n $num | grep "^ $a$" +fi +done | wc -l ) +echo $number +if [ $ag="yes" ]; then +echo "labels += [\" $a \" + '(' + str( $number ) + ')' ];numbers += [ $number + .2 ]" | egrep -v ' NgIohTuned | NGDSRW | AX | AXP | PCABO | PSO | PymooBIPOP | CMA | LognormalDiscreteOnePlusOne ' +else +echo "labels += [\" $a \" + '(' + str( $number ) + ')' ];numbers += [ $number + .2 ]" +fi +done +echo "plt.pie(numbers, labels=labels)" +echo "plt.title(title)" +if [ $ag="yes" ]; then +echo "plt.savefig(f'agpie{num}.png')" +else +echo "plt.savefig(f'pie{num}.png')" +fi +done +done +) > plotpie.py + +python plotpie.py +#sed -i 's/label.* AX .*//g' plotpie.py +#sed -i 's/label.* PSO .*//g' plotpie.py +#sed -i 's/label.* PymooBIPOP .*//g' plotpie.py +#sed -i 's/label.* CMA .*//g' plotpie.py +#sed -i 's/pie{num}/agpie{num}/g' plotpie.py +# +#python plotpie.py diff --git a/scripts/plot_dagstuhloid.sh b/scripts/plot_dagstuhloid.sh index 45f5b393e..32caa4244 100755 --- a/scripts/plot_dagstuhloid.sh +++ b/scripts/plot_dagstuhloid.sh @@ -7,6 +7,8 @@ #SBATCH --nodes=1 #SBATCH --cpus-per-task=67 + + # This script works both as a standalone script or with slurm. # This is much slower than mini_plot_dagstuhloid, but outputs more data (in particular: competence maps and plots for # subcases). @@ -18,11 +20,14 @@ if compgen -G "*.csv" > /dev/null; then # First we run all nevergrad plotting. for i in `ls *.csv ` do - (python -m nevergrad.benchmark.plotting --nomanyxp=1 $i ; python -m nevergrad.benchmark.plotting --max_combsize=2 --competencemaps=1 --nomanyxp=1 $i ) & + python -m nevergrad.benchmark.plotting --nomanyxp=1 $i & + #(python -m nevergrad.benchmark.plotting --nomanyxp=1 $i ; python -m nevergrad.benchmark.plotting --max_combsize=2 --competencemaps=1 --nomanyxp=1 $i ) & done wait +./scripts/multirank.sh +./scripts/get_statistics.sh # Second we do pdflatex for i in *.csv @@ -43,5 +48,5 @@ fi # End of "there is something to do". # tar -zcvf ~/dag.tgz *_plots scripts/latexize.sh -tar -zcvf dagstuhloid.tgz dagstuhloid.pdf *.csv *plots/xpresults_all.png rnk_*.txt *plots/fight_all.png.cp.txt +tar -zcvf dagstuhloid.tgz dagstuhloid.pdf *.csv *plots/xpresults_all.png rnk_*.txt *plots/fight_all.png.cp.txt pie*.png agpi*.png diff --git a/scripts/tex/end.tex b/scripts/tex/end.tex index f7ff776f2..f3738a72e 100644 --- a/scripts/tex/end.tex +++ b/scripts/tex/end.tex @@ -1,4 +1,19 @@ + +\section{Global pies} + +\subsection{Including all} +\includegraphics[width=.48\textwidth]{pie1.png} +\includegraphics[width=.48\textwidth]{pie2.png} +\includegraphics[width=.48\textwidth]{pie4.png} +\includegraphics[width=.48\textwidth]{pie8.png} +\subsection{No NgIohTuned, no NGDSRW, only one per category} +\includegraphics[width=.48\textwidth]{pie1.png} +\includegraphics[width=.48\textwidth]{pie2.png} +\includegraphics[width=.48\textwidth]{pie4.png} +\includegraphics[width=.48\textwidth]{pie8.png} + + \section{Comparison with baseline} \subsection{Comparison for the normalized simple regret} \input{compa.tex} From 1eab5b0782ca8a808a71929c09e04e7fa7578197 Mon Sep 17 00:00:00 2001 From: Olivier Teytaud Date: Sat, 12 Oct 2024 15:08:51 +0200 Subject: [PATCH 3/8] fix --- nevergrad/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nevergrad/__init__.py b/nevergrad/__init__.py index f60f1139a..1daff4cfa 100644 --- a/nevergrad/__init__.py +++ b/nevergrad/__init__.py @@ -15,4 +15,4 @@ __all__ = ["optimizers", "families", "callbacks", "p", "typing", "errors", "ops"] -__version__ = "1.0.4" +__version__ = "1.0.5" From fb82d44f315ff11469e028a3d8d9cadaaa5f9f75 Mon Sep 17 00:00:00 2001 From: Olivier Teytaud Date: Sat, 12 Oct 2024 15:11:07 +0200 Subject: [PATCH 4/8] clean --- nevergrad/benchmark/xpbase.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nevergrad/benchmark/xpbase.py b/nevergrad/benchmark/xpbase.py index d0327a104..2e645b380 100644 --- a/nevergrad/benchmark/xpbase.py +++ b/nevergrad/benchmark/xpbase.py @@ -12,7 +12,6 @@ import nevergrad.common.typing as ngtp import typing as tp import numpy as np -from nevergrad.optimization import callbacks as ngcallbacks from nevergrad.parametrization import parameter as p from nevergrad.common import decorators from nevergrad.common import errors From b6f2c8dfe5c8ea71ca1a1932c48342dc295505f7 Mon Sep 17 00:00:00 2001 From: Olivier Teytaud Date: Sat, 12 Oct 2024 15:53:58 +0200 Subject: [PATCH 5/8] po --- nevergrad/benchmark/experiments.py | 60 +++++----- nevergrad/benchmark/gymexperiments.py | 158 ++++++++++++------------- nevergrad/optimization/optimizerlib.py | 4 +- nevergrad/optimization/recaster.py | 1 + scripts/latexize.sh | 2 +- 5 files changed, 113 insertions(+), 112 deletions(-) diff --git a/nevergrad/benchmark/experiments.py b/nevergrad/benchmark/experiments.py index a9616b930..56225b0d3 100644 --- a/nevergrad/benchmark/experiments.py +++ b/nevergrad/benchmark/experiments.py @@ -54,36 +54,36 @@ # list_optims = ["QOTPDE", "LQOTPDE", "LQODE"] # list_optims = ["SPQODE", "SQOPSO", "DiagonalCMA"] def refactor_optims(x: tp.List[tp.Any]) -> tp.List[tp.Any]: # type: ignore - return list( - np.random.choice( - [ - "RandomSearch", - "AXP", - "LognormalDiscreteOnePlusOne", - "AX", - "CMA", - "Cobyla", - "PCABO", - "SMAC3", - "NgIohTuned", - "NGDSRW", - "PymooBIPOP", - "CMA", - "PSO", - "SQOPSO", - "DE", - "DiscreteLenglerOnePlusOne", - "DiscreteOnePlusOne", - "OnePlusOne", - "DSproba", - "MetaModel", - ], - 1, - ) - ) - return [ - np.random.choice(["RandomSearch", "AX", "CMA", "Cobyla", "PCABO", "SMAC3", "NgIohTuned", "NGDSRW"]) - ] + # return list( + # np.random.choice( + # [ + # "RandomSearch", + # "AXP", + # "LognormalDiscreteOnePlusOne", + # "AX", + # "CMA", + # "Cobyla", + # "PCABO", + # "SMAC3", + # "NgIohTuned", + # "NGDSRW", + # "PymooBIPOP", + # "CMA", + # "PSO", + # "SQOPSO", + # "DE", + # "DiscreteLenglerOnePlusOne", + # "DiscreteOnePlusOne", + # "OnePlusOne", + # "DSproba", + # "MetaModel", + # ], + # 1, + # ) + # ) + # return [ + # np.random.choice(["RandomSearch", "AX", "CMA", "Cobyla", "PCABO", "SMAC3", "NgIohTuned", "NGDSRW"]) + # ] # return ["NgIoh4", "NgIoh21", "NgIohTuned", "NgIohLn", "NgIohRS", "LBFGSB"] if False: # np.random.randn() < 0.0: return list( diff --git a/nevergrad/benchmark/gymexperiments.py b/nevergrad/benchmark/gymexperiments.py index f8f8ba2e9..868265f85 100644 --- a/nevergrad/benchmark/gymexperiments.py +++ b/nevergrad/benchmark/gymexperiments.py @@ -22,85 +22,85 @@ def gym_problem_modifier(specific_problem): def gym_optimizer_modifier(optims): - return list( - np.random.choice( - [ - "RandomSearch", - "AXP", - "LognormalDiscreteOnePlusOne", - "AX", - "CMA", - "Cobyla", - "PCABO", - "SMAC3", - "NgIohTuned", - "NGDSRW", - "PymooBIPOP", - "CMA", - "PSO", - "SQOPSO", - "DE", - "DiscreteLenglerOnePlusOne", - "DiscreteOnePlusOne", - "OnePlusOne", - "DSproba", - "MetaModel", - ], - 1, - ) - ) - return list( - np.random.choice( - [ - "RandomSearch", - "AX", - "CMA", - "Cobyla", - "PCABO", - "SMAC3", - "NgIohTuned", - "NGDSRW", - "PymooBIPOP", - "CMA", - "PSO", - "SQOPSO", - "DE", - "DiscreteLenglerOnePlusOne", - "DiscreteOnePlusOne", - "OnePlusOne", - "DSproba", - "MetaModel", - ], - 3, - ) - ) - return list( - np.random.choice( - [ - "RandomSearch", - "AXP", - "LognormalDiscreteOnePlusOne", - "AX", - "CMA", - "Cobyla", - "PCABO", - "SMAC3", - "NgIohTuned", - "NGDSRW", - "PymooBIPOP", - "CMA", - "PSO", - "SQOPSO", - "DE", - "DiscreteLenglerOnePlusOne", - "DiscreteOnePlusOne", - "OnePlusOne", - "DSproba", - "MetaModel", - ], - 1, - ) - ) + # return list( + # np.random.choice( + # [ + # "RandomSearch", + # "AXP", + # "LognormalDiscreteOnePlusOne", + # "AX", + # "CMA", + # "Cobyla", + # "PCABO", + # "SMAC3", + # "NgIohTuned", + # "NGDSRW", + # "PymooBIPOP", + # "CMA", + # "PSO", + # "SQOPSO", + # "DE", + # "DiscreteLenglerOnePlusOne", + # "DiscreteOnePlusOne", + # "OnePlusOne", + # "DSproba", + # "MetaModel", + # ], + # 1, + # ) + # ) + # return list( + # np.random.choice( + # [ + # "RandomSearch", + # "AX", + # "CMA", + # "Cobyla", + # "PCABO", + # "SMAC3", + # "NgIohTuned", + # "NGDSRW", + # "PymooBIPOP", + # "CMA", + # "PSO", + # "SQOPSO", + # "DE", + # "DiscreteLenglerOnePlusOne", + # "DiscreteOnePlusOne", + # "OnePlusOne", + # "DSproba", + # "MetaModel", + # ], + # 3, + # ) + # ) + # return list( + # np.random.choice( + # [ + # "RandomSearch", + # "AXP", + # "LognormalDiscreteOnePlusOne", + # "AX", + # "CMA", + # "Cobyla", + # "PCABO", + # "SMAC3", + # "NgIohTuned", + # "NGDSRW", + # "PymooBIPOP", + # "CMA", + # "PSO", + # "SQOPSO", + # "DE", + # "DiscreteLenglerOnePlusOne", + # "DiscreteOnePlusOne", + # "OnePlusOne", + # "DSproba", + # "MetaModel", + # ], + # 1, + # ) + # ) print(optims) if os.environ.get("GYM_OPTIMIZER") is not None: optimizer_string = os.environ.get("GYM_OPTIMIZER") diff --git a/nevergrad/optimization/optimizerlib.py b/nevergrad/optimization/optimizerlib.py index b8cade071..0f589c7f4 100644 --- a/nevergrad/optimization/optimizerlib.py +++ b/nevergrad/optimization/optimizerlib.py @@ -1164,7 +1164,7 @@ def __init__( ) -> None: super().__init__(parametrization, budget=budget, num_workers=num_workers) try: - from ax.service.ax_client import AxClient, ObjectiveProperties + from ax.service.ax_client import AxClient, ObjectiveProperties # type: ignore except Exception as e: print(f"Pb for creating AX solver") raise e @@ -1193,7 +1193,7 @@ def p(x): trial_index_to_param, _ = self.ax_client.get_next_trials(max_trials=1) assert len(trial_index_to_param) == 1 vals = np.zeros(self.dimension) - for a, _trial in trial_index_to_param.items(): + for _, _trial in trial_index_to_param.items(): trial = _trial for i in range(self.dimension): vals[i] = invsig(trial["x" + str(i)]) diff --git a/nevergrad/optimization/recaster.py b/nevergrad/optimization/recaster.py index ff411a8a1..aa0c07e47 100644 --- a/nevergrad/optimization/recaster.py +++ b/nevergrad/optimization/recaster.py @@ -414,6 +414,7 @@ def minimize( batch_mode: bool = False, verbosity: int = 0, constraint_violation: tp.Any = None, + max_time: tp.Optional[float] = None, ) -> p.Parameter: raise NotImplementedError("This optimizer isn't supported by the way minimize works by default.") diff --git a/scripts/latexize.sh b/scripts/latexize.sh index 3d3aaf2bf..a02a26f90 100755 --- a/scripts/latexize.sh +++ b/scripts/latexize.sh @@ -176,7 +176,7 @@ cat tmp.tex.tmp #ls `ls $v | sed 's/\.tex/\.pdf/g'` | sed 's/.*/\\includegraphics[width=.99\\textwidth]{{&}}\\\\/g' #done #done -echo '\section{Agregated table}' +echo '\section{Aggregated table}' cat agtable.tex cat scripts/tex/end.tex ) >> dagstuhloid.tex for v in competition.tex dagstuhloid.tex From e918a91b7521eadfff0bc75c0c8ae905e869d95b Mon Sep 17 00:00:00 2001 From: Olivier Teytaud Date: Tue, 19 Nov 2024 10:43:13 +0200 Subject: [PATCH 6/8] retm --- nevergrad/benchmark/experiments.py | 62 +++++++++++++----------- nevergrad/benchmark/gymexperiments.py | 11 +++++ nevergrad/benchmark/plotting.py | 32 +++++++------ nevergrad/benchmark/xpbase.py | 6 +++ nevergrad/optimization/optimizerlib.py | 20 ++++++-- scripts/dagstuhloid.sh | 7 ++- scripts/get_statistics.sh | 4 +- scripts/multirank.sh | 17 ++++--- scripts/multirank_robust.sh | 65 ++++++++++++++++++++++++++ scripts/plot_dagstuhloid.sh | 10 ++-- scripts/tex/end.tex | 12 ++--- 11 files changed, 176 insertions(+), 70 deletions(-) create mode 100755 scripts/multirank_robust.sh diff --git a/nevergrad/benchmark/experiments.py b/nevergrad/benchmark/experiments.py index 56225b0d3..f97464abf 100644 --- a/nevergrad/benchmark/experiments.py +++ b/nevergrad/benchmark/experiments.py @@ -54,33 +54,41 @@ # list_optims = ["QOTPDE", "LQOTPDE", "LQODE"] # list_optims = ["SPQODE", "SQOPSO", "DiagonalCMA"] def refactor_optims(x: tp.List[tp.Any]) -> tp.List[tp.Any]: # type: ignore - # return list( - # np.random.choice( - # [ - # "RandomSearch", - # "AXP", - # "LognormalDiscreteOnePlusOne", - # "AX", - # "CMA", - # "Cobyla", - # "PCABO", - # "SMAC3", - # "NgIohTuned", - # "NGDSRW", - # "PymooBIPOP", - # "CMA", - # "PSO", - # "SQOPSO", - # "DE", - # "DiscreteLenglerOnePlusOne", - # "DiscreteOnePlusOne", - # "OnePlusOne", - # "DSproba", - # "MetaModel", - # ], - # 1, - # ) - # ) + #return ["SMAC3"] + #return ["AXP"] + #return [ "RandomSearch","Cobyla","NgIohTuned","PymooBIPOP","CMA","PSO","SQOPSO","DE","DiscreteLenglerOnePlusOne","DiscreteOnePlusOne","OnePlusOne","DSproba","MetaModel","LognormalDiscreteOnePlusOne","CauchyRandomSearch","RandomScaleRandomSearchPlusMiddlePoint","HullAvgMetaTuneRecentering","HyperOpt","NGDSRW"] + return [np.random.choice([ "RandomSearch","AXP","Cobyla","PCABO","SMAC3","NgIohTuned","PymooBIPOP","CMA","PSO","SQOPSO","DE","DiscreteLenglerOnePlusOne","DiscreteOnePlusOne","OnePlusOne","DSproba","MetaModel","LognormalDiscreteOnePlusOne","CauchyRandomSearch","RandomScaleRandomSearchPlusMiddlePoint","HullAvgMetaTuneRecentering","HyperOpt","NGDSRW"])] + return ["RandomSearch"] + return ["HyperOpt"] + return ["CauchyRandomSearch"] + return ["RandomScaleRandomSearchPlusMiddlePoint"] + return ["HullAvgMetaTuneRecentering"] + return list( + np.random.choice( + [ + "RandomSearch", + "AXP", + "LognormalDiscreteOnePlusOne", + "CMA", + "Cobyla", + "PCABO", + "SMAC3", + "NgIohTuned", + "NGDSRW", + "PymooBIPOP", + "CMA", + "PSO", + "SQOPSO", + "DE", + "DiscreteLenglerOnePlusOne", + "DiscreteOnePlusOne", + "OnePlusOne", + "DSproba", + "MetaModel", + ], + 1, + ) + ) # return [ # np.random.choice(["RandomSearch", "AX", "CMA", "Cobyla", "PCABO", "SMAC3", "NgIohTuned", "NGDSRW"]) # ] diff --git a/nevergrad/benchmark/gymexperiments.py b/nevergrad/benchmark/gymexperiments.py index 868265f85..4eca2a8cb 100644 --- a/nevergrad/benchmark/gymexperiments.py +++ b/nevergrad/benchmark/gymexperiments.py @@ -3,6 +3,7 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. +import numpy as np import os import typing as tp from nevergrad.functions import gym as nevergrad_gym @@ -22,6 +23,16 @@ def gym_problem_modifier(specific_problem): def gym_optimizer_modifier(optims): + return [np.random.choice([ "RandomSearch","AXP","Cobyla","PCABO","SMAC3","NgIohTuned","PymooBIPOP","CMA","PSO","SQOPSO","DE","DiscreteLenglerOnePlusOne","DiscreteOnePlusOne","OnePlusOne","DSproba","MetaModel","LognormalDiscreteOnePlusOne","CauchyRandomSearch","RandomScaleRandomSearchPlusMiddlePoint","HullAvgMetaTuneRecentering","HyperOpt","NGDSRW"])] + return [ "RandomSearch","Cobyla","NgIohTuned","PymooBIPOP","CMA","PSO","SQOPSO","DE","DiscreteLenglerOnePlusOne","DiscreteOnePlusOne","OnePlusOne","DSproba","MetaModel","LognormalDiscreteOnePlusOne","CauchyRandomSearch","RandomScaleRandomSearchPlusMiddlePoint","HullAvgMetaTuneRecentering","HyperOpt","NGDSRW"] + return [ "RandomSearch","AXP","Cobyla","PCABO","SMAC3","NgIohTuned","PymooBIPOP","CMA","PSO","SQOPSO","DE","DiscreteLenglerOnePlusOne","DiscreteOnePlusOne","OnePlusOne","DSproba","MetaModel","LognormalDiscreteOnePlusOne","CauchyRandomSearch","RandomScaleRandomSearchPlusMiddlePoint","HullAvgMetaTuneRecentering","HyperOpt","NGDSRW"] + return ["RandomSearch"] + return ["HyperOpt"] + return ["CauchyRandomSearch"] + return ["RandomScaleRandomSearchPlusMiddlePoint"] + return ["HullAvgMetaTuneRecentering"] + return ["SMAC3"] + return ["AXP"] # return list( # np.random.choice( # [ diff --git a/nevergrad/benchmark/plotting.py b/nevergrad/benchmark/plotting.py index 9f5e70420..46d3f04c5 100644 --- a/nevergrad/benchmark/plotting.py +++ b/nevergrad/benchmark/plotting.py @@ -402,6 +402,12 @@ def create_plots( name = "fight_all.png" if name == "fight_.png" else name name = compactize(name) fullname = name + if name == "fight_all.png": + with open(str(output_folder / name) + ".cp.txt", "w") as f: + f.write(fullname) + f.write("ranking:\n") + for i, algo in enumerate(data_df.columns[:158]): + f.write(f" algo {i}: {algo}\n") if len(name) > 240: hashcode = hashlib.md5(bytes(name, "utf8")).hexdigest() @@ -415,13 +421,8 @@ def create_plots( ) fplotter = FightPlotter(data_df) if name == "fight_all.png": - with open(str(output_folder / name) + ".cp.txt", "w") as f: - f.write(fullname) - f.write("ranking:\n") - global pure_algorithms - pure_algorithms = list(data_df.columns[:]) - for i, algo in enumerate(data_df.columns[:158]): - f.write(f" algo {i}: {algo}\n") + global pure_algorithms + pure_algorithms = list(data_df.columns[:]) if name == "fight_all.png": fplotter.save(str(output_folder / "fight_all_pure.png"), dpi=_DPI) else: @@ -445,9 +446,10 @@ def create_plots( out_filepath = output_folder / "xpresults_all.png" try: data = XpPlotter.make_data(df, normalized_loss=True) - xpplotter = XpPlotter( - data, title=os.path.basename(output_folder), name_style=name_style, xaxis=xpaxis, pure_only=True - ) + for pure_only in [False, True]: + xpplotter = XpPlotter( + data, title=os.path.basename(output_folder), name_style=name_style, xaxis=xpaxis, pure_only=pure_only + ) except Exception as e: lower = 0 upper = len(df) @@ -622,11 +624,11 @@ def __init__( sorted_optimizers = [ o for o in sorted_optimizers if o + " " in [p[: (len(o) + 1)] for p in pure_algorithms] ] - with open("rnk__" + str(title) + ".cp.txt", "w") as f: - f.write(compactize(title)) - f.write("ranking:\n") - for i, algo in reversed(list(enumerate(sorted_optimizers))): - f.write(f" algo {i}: {algo} (x)\n") + with open(("rnk__" if not pure_only else "rnkpure__") + str(title) + ".cp.txt", "w") as f: + f.write(compactize(title)) + f.write("ranking:\n") + for i, algo in reversed(list(enumerate(sorted_optimizers))): + f.write(f" algo {i}: {algo} (x)\n") # print(sorted_optimizers, " merged with ", pure_algorithms) # print("Leads to ", sorted_optimizers) self._fig = plt.figure() diff --git a/nevergrad/benchmark/xpbase.py b/nevergrad/benchmark/xpbase.py index 2e645b380..ec6ce405e 100644 --- a/nevergrad/benchmark/xpbase.py +++ b/nevergrad/benchmark/xpbase.py @@ -291,6 +291,11 @@ def _run_with_error(self, callbacks: tp.Optional[tp.Dict[str, obase._OptimCallBa try: # call the actual Optimizer.minimize method because overloaded versions could alter the worklflow # and provide unfair comparisons (especially for parallelized settings) + import signal + def handler(signum, frame): + raise Exception("Time exceeded!") + signal.signal(signal.SIGALRM, handler) + signal.alarm(int(2.8 * 3600 * 24)) if self.suggestions is not None: for s in self.suggestions: self._optimizer.suggest(s) @@ -302,6 +307,7 @@ def _run_with_error(self, callbacks: tp.Optional[tp.Dict[str, obase._OptimCallBa constraint_violation=self.constraint_violation, max_time=3600 * 24 * 2.5, ) + signal.alarm(0) except Exception as e: # pylint: disable=broad-except self._log_results(pfunc, t0, self._optimizer.num_ask) raise e diff --git a/nevergrad/optimization/optimizerlib.py b/nevergrad/optimization/optimizerlib.py index 0f589c7f4..0be185989 100644 --- a/nevergrad/optimization/optimizerlib.py +++ b/nevergrad/optimization/optimizerlib.py @@ -1178,6 +1178,7 @@ def __init__( objectives={"result": ObjectiveProperties(minimize=True)}, # parameter_constraints=["x + y <= 2.0"], # Optional. ) + self._trials = [] # def _internal_provide_recommendation( # self, @@ -1187,14 +1188,16 @@ def _internal_ask_candidate(self) -> p.Parameter: def invsig(x): def p(x): return np.clip(x, 1e-15, 1.0 - 1e-15) - return np.log(p(x) / (1 - p(x))) - trial_index_to_param, _ = self.ax_client.get_next_trials(max_trials=1) - assert len(trial_index_to_param) == 1 + if len(self._trials) == 0: + trial_index_to_param, _ = self.ax_client.get_next_trials(max_trials=1) + for _, _trial in trial_index_to_param.items(): + trial = _trial + self._trials += [trial] + trial = self._trials[0] + self._trials = self._trials[1:] vals = np.zeros(self.dimension) - for _, _trial in trial_index_to_param.items(): - trial = _trial for i in range(self.dimension): vals[i] = invsig(trial["x" + str(i)]) candidate = self.parametrization.spawn_child().set_standardized_data(vals) @@ -3974,6 +3977,13 @@ def _select_optimizer_cls(self) -> base.OptCls: return optCls +#@registry.register +#class MetaBO(NGOptBase): +# """Nevergrad optimizer by competence map. You might modify this one for designing your own competence map.""" +# +# def _select_optimizer_cls(self) -> base.OptCls: +# optCls: base.OptCls = NGOptBase +# funcinfo = self.parametrization.function @registry.register class NGO(NGOptBase): # compatibility pass diff --git a/scripts/dagstuhloid.sh b/scripts/dagstuhloid.sh index 848b13364..fd0af9588 100755 --- a/scripts/dagstuhloid.sh +++ b/scripts/dagstuhloid.sh @@ -6,10 +6,13 @@ #SBATCH --partition=scavenge #SBATCH --nodes=1 #SBATCH --cpus-per-task=70 -#SBATCH -a 0-323 +#SBATCH -a 0-76 -tasks=(nozp_noms_bbob multi_structured_ng_full_gym lsgo smallbudget_lsgo ng_gym sequential_fastgames small_deterministic_gym_multi tiny_deterministic_gym_multi zp_pbbob zp_ms_bbob aquacrop_fao bonnans deceptive double_o_seven fishing instrum_discrete keras_tuning mldakmeans mltuning mono_rocket multimodal multiobjective_example multiobjective_example_hd multiobjective_example_many_hd naive_seq_keras_tuning naive_seq_mltuning naive_veryseq_keras_tuning naivemltuning nano_naive_seq_mltuning nano_naive_veryseq_mltuning nano_seq_mltuning nano_veryseq_mltuning neuro_oneshot_mltuning pbbob pbo_reduced_suite reduced_yahdlbbbob rocket seq_keras_tuning seq_mltuning sequential_instrum_discrete sequential_topology_optimization spsa_benchmark topology_optimization ultrasmall_photonics ultrasmall_photonics2 veryseq_keras_tuning yabbob yabigbbob yaboundedbbob yaboxbbob yahdbbob yahdnoisybbob yamegapenbbob yamegapenboundedbbob yamegapenboxbbob yanoisybbob yaonepenbbob yaonepenboundedbbob yaonepenboxbbob yaonepennoisybbob yaonepenparabbob yaonepensmallbbob yaparabbob yapenbbob yapenboundedbbob yapenboxbbob yapennoisybbob yapenparabbob yapensmallbbob yasmallbbob yatinybbob yatuningbbob ms_bbob ranknoisy powersystems verysmall_photonics verysmall_photonics2) +tasks=(nozp_noms_bbob multi_structured_ng_full_gym lsgo smallbudget_lsgo ng_gym sequential_fastgames small_deterministic_gym_multi tiny_deterministic_gym_multi zp_pbbob zp_ms_bbob aquacrop_fao bonnans deceptive double_o_seven fishing instrum_discrete keras_tuning mldakmeans mltuning mono_rocket multimodal multiobjective_example multiobjective_example_hd multiobjective_example_many_hd naive_seq_keras_tuning naive_seq_mltuning naive_veryseq_keras_tuning naivemltuning nano_naive_seq_mltuning nano_naive_veryseq_mltuning nano_seq_mltuning nano_veryseq_mltuning neuro_oneshot_mltuning pbbob pbo_reduced_suite reduced_yahdlbbbob rocket seq_keras_tuning seq_mltuning sequential_instrum_discrete sequential_topology_optimization spsa_benchmark topology_optimization ultrasmall_photonics ultrasmall_photonics2 veryseq_keras_tuning yahdlbbbob yabbob yabigbbob yaboundedbbob yaboxbbob yahdbbob yahdnoisybbob yamegapenbbob yamegapenboundedbbob yamegapenboxbbob yanoisybbob yaonepenbbob yaonepenboundedbbob yaonepenboxbbob yaonepennoisybbob yaonepenparabbob yaonepensmallbbob yaparabbob yapenbbob yapenboundedbbob yapenboxbbob yapennoisybbob yapenparabbob yapensmallbbob yasmallbbob yatinybbob yatuningbbob ms_bbob ranknoisy powersystems verysmall_photonics verysmall_photonics2) +#tasks=(keras_tuning lsgo mono_rocket naive_seq_keras_tuning naive_seq_mltuning naive_veryseq_keras_tuning nano_naive_seq_mltuning nano_seq_mltuning ranknoisy rocket seq_keras_tuning seq_mltuning veryseq_keras_tuning yahdnoisybbob yaonepennoisybbob yapennoisybbob ) +# TODO remove above + #tasks=(ng_gym small_deterministic_gym_multi tiny_deterministic_gym_multi zp_pbbob zp_ms_bbob aquacrop_fao bonnans deceptive double_o_seven fishing instrum_discrete keras_tuning mldakmeans mltuning mono_rocket multimodal multiobjective_example multiobjective_example_hd multiobjective_example_many_hd naive_seq_keras_tuning naive_seq_mltuning naive_veryseq_keras_tuning naivemltuning nano_naive_seq_mltuning nano_naive_veryseq_mltuning nano_seq_mltuning nano_veryseq_mltuning neuro_oneshot_mltuning pbbob pbo_reduced_suite reduced_yahdlbbbob rocket seq_keras_tuning seq_mltuning sequential_instrum_discrete sequential_topology_optimization spsa_benchmark topology_optimization ultrasmall_photonics ultrasmall_photonics2 veryseq_keras_tuning yabbob yabigbbob yaboundedbbob yaboxbbob yahdbbob yahdnoisybbob yamegapenbbob yamegapenboundedbbob yamegapenboxbbob yanoisybbob yaonepenbbob yaonepenboundedbbob yaonepenboxbbob yaonepennoisybbob yaonepenparabbob yaonepensmallbbob yaparabbob yapenbbob yapenboundedbbob yapenboxbbob yapennoisybbob yapenparabbob yapensmallbbob yasmallbbob yatinybbob yatuningbbob ms_bbob ranknoisy powersystems verysmall_photonics verysmall_photonics2) task=${tasks[SLURM_ARRAY_TASK_ID % ${#tasks[@]} ]} diff --git a/scripts/get_statistics.sh b/scripts/get_statistics.sh index 04479796e..23397f5cf 100755 --- a/scripts/get_statistics.sh +++ b/scripts/get_statistics.sh @@ -3,9 +3,9 @@ pbs=`ls rnk*.txt | sed 's/rnk__//g' | sed 's/_plots.*//g'` ( -echo '\begin{tabular}{|c|c|c|c|}' +echo '\begin{tabular}{|p{2cm}|c|c|c|}' echo '\hline' -echo 'Best algorithm & Dimension & Budget & Problem name \\' +echo 'Best algorithm & Problem name & Dimension & Budget \\' echo '\hline' for pb in $pbs diff --git a/scripts/multirank.sh b/scripts/multirank.sh index dee57071d..b8f7d23c9 100755 --- a/scripts/multirank.sh +++ b/scripts/multirank.sh @@ -1,5 +1,8 @@ #!/bin/bash +echo DEPRECATED + +exit 1 ( echo 'import matplotlib' echo 'import matplotlib.pyplot as plt' @@ -22,28 +25,28 @@ echo 'plt.clf()' echo "labels = []" echo "numbers = []" echo "# $num best" -for a in RandomSearch AX AXP Cobyla PCABO SMAC3 NgIohTuned NGDSRW PymooBIPOP CMA PSO SQOPSO DE DiscreteLenglerOnePlusOne DiscreteOnePlusOne OnePlusOne DSproba MetaModel LognormalDiscreteOnePlusOne +for a in RandomSearch AXP Cobyla PCABO SMAC3 NgIohTuned PymooBIPOP CMA PSO SQOPSO DE DiscreteLenglerOnePlusOne DiscreteOnePlusOne OnePlusOne DSproba MetaModel LognormalDiscreteOnePlusOne do echo -n "# $a is in the $num best in this number of problems:" number=$( for k in rnk*.txt do -if [ $ag="yes" ]; then - grep ':' $k | sed 's/.*://g' | sed 's/ (.*//g' | egrep '[a-zA-Z0-9]' | egrep -v ' NgIohTuned | NGDSRW | AX | AXP | LognormalDiscreteOnePlusOne | PCABO | PSO | PymooBIPOP | CMA ' | head -n $num | grep "^ $a$" +if [ "$ag" == yes ]; then + grep ':' $k | grep -v ' AX ' | sed 's/.*://g' | sed 's/ (.*//g' | egrep '[a-zA-Z0-9]' | egrep -v ' NgIohTuned | NGDSRW | AX | SMAC3 | LognormalDiscreteOnePlusOne | PCABO | PSO | PymooBIPOP | CMA ' | head -n $num | grep "^ $a$" else - grep ':' $k | sed 's/.*://g' | sed 's/ (.*//g' | egrep '[a-zA-Z0-9]' | head -n $num | grep "^ $a$" + grep ':' $k | grep -v ' AX ' | grep -v ' NGDSRW ' | sed 's/.*://g' | sed 's/ (.*//g' | egrep '[a-zA-Z0-9]' | head -n $num | grep "^ $a$" fi done | wc -l ) echo $number -if [ $ag="yes" ]; then -echo "labels += [\" $a \" + '(' + str( $number ) + ')' ];numbers += [ $number + .2 ]" | egrep -v ' NgIohTuned | NGDSRW | AX | AXP | PCABO | PSO | PymooBIPOP | CMA | LognormalDiscreteOnePlusOne ' +if [ "$ag" == yes ]; then +echo "labels += [\" $a \" + '(' + str( $number ) + ')' ];numbers += [ $number + .2 ]" | egrep -v ' NgIohTuned | NGDSRW | AX | SMAC3 | PCABO | PSO | PymooBIPOP | CMA | LognormalDiscreteOnePlusOne ' else echo "labels += [\" $a \" + '(' + str( $number ) + ')' ];numbers += [ $number + .2 ]" fi done echo "plt.pie(numbers, labels=labels)" echo "plt.title(title)" -if [ $ag="yes" ]; then +if [ "$ag" == yes ]; then echo "plt.savefig(f'agpie{num}.png')" else echo "plt.savefig(f'pie{num}.png')" diff --git a/scripts/multirank_robust.sh b/scripts/multirank_robust.sh new file mode 100755 index 000000000..fcfb31fd5 --- /dev/null +++ b/scripts/multirank_robust.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +echo DEPRECATED + +exit 1 +( +echo 'import matplotlib' +echo 'import matplotlib.pyplot as plt' + +#for k in rnk*.txt +#do +#echo -n "# $k :" +#grep ':' $k | sed 's/.*://g' | sed 's/ (.*//g' | egrep '[a-zA-Z0-9]' | sort | uniq | wc -l +# +#done + +for ag in yes no +do +for num in 1 2 4 8 +do +echo "num=$num" +echo "numt=$( ls */*.cp.txt | wc -l )" +echo "title=f'How many times in the {num} best ? out of {numt}'" +echo 'plt.clf()' +echo "labels = []" +echo "numbers = []" +echo "# $num best" +for a in RandomSearch AXP Cobyla PCABO SMAC3 NgIohTuned PymooBIPOP CMA PSO SQOPSO DE DiscreteLenglerOnePlusOne DiscreteOnePlusOne OnePlusOne DSproba MetaModel LognormalDiscreteOnePlusOne +do +echo -n "# $a is in the $num best in this number of problems:" +number=$( +for k in */*.cp.txt +do +if [ "$ag" == yes ]; then + grep ':' $k | grep -v ' AX ' | sed 's/.*://g' | sed 's/ (.*//g' | egrep '[a-zA-Z0-9]' | egrep -v ' NgIohTuned | NGDSRW | AX | SMAC3 | LognormalDiscreteOnePlusOne | PCABO | PSO | PymooBIPOP | CMA ' | head -n $num | grep "^ $a$" +else + grep ':' $k | grep -v ' AX ' | grep -v 'NGDSRW ' | sed 's/.*://g' | sed 's/ (.*//g' | egrep '[a-zA-Z0-9]' | head -n $num | grep "^ $a$" +fi +done | wc -l ) +echo $number +if [ "$ag" == yes ]; then +echo "labels += [\" $a \" + '(' + str( $number ) + ')' ];numbers += [ $number + .2 ]" | egrep -v ' NgIohTuned | NGDSRW | AX | SMAC3 | PCABO | PSO | PymooBIPOP | CMA | LognormalDiscreteOnePlusOne ' +else +echo "labels += [\" $a \" + '(' + str( $number ) + ')' ];numbers += [ $number + .2 ]" +fi +done +echo "plt.pie(numbers, labels=labels)" +echo "plt.title(title)" +if [ "$ag" == "yes" ]; then +echo "plt.savefig(f'agpierob{num}.png')" +else +echo "plt.savefig(f'pierob{num}.png')" +fi +done +done +) > plotpie.py + +python plotpie.py +#sed -i 's/label.* AX .*//g' plotpie.py +#sed -i 's/label.* PSO .*//g' plotpie.py +#sed -i 's/label.* PymooBIPOP .*//g' plotpie.py +#sed -i 's/label.* CMA .*//g' plotpie.py +#sed -i 's/pie{num}/agpie{num}/g' plotpie.py +# +#python plotpie.py diff --git a/scripts/plot_dagstuhloid.sh b/scripts/plot_dagstuhloid.sh index 32caa4244..612b6b4a2 100755 --- a/scripts/plot_dagstuhloid.sh +++ b/scripts/plot_dagstuhloid.sh @@ -20,13 +20,15 @@ if compgen -G "*.csv" > /dev/null; then # First we run all nevergrad plotting. for i in `ls *.csv ` do - python -m nevergrad.benchmark.plotting --nomanyxp=1 $i & - #(python -m nevergrad.benchmark.plotting --nomanyxp=1 $i ; python -m nevergrad.benchmark.plotting --max_combsize=2 --competencemaps=1 --nomanyxp=1 $i ) & + #python -m nevergrad.benchmark.plotting --nomanyxp=1 $i & + (python -m nevergrad.benchmark.plotting --nomanyxp=1 $i ; python -m nevergrad.benchmark.plotting --max_combsize=2 --competencemaps=1 --nomanyxp=1 $i ) & done wait -./scripts/multirank.sh +# ./scripts/multirank.sh +# ./scripts/multirank_robust.sh +./scripts/pies_multirank.sh ./scripts/get_statistics.sh # Second we do pdflatex @@ -48,5 +50,5 @@ fi # End of "there is something to do". # tar -zcvf ~/dag.tgz *_plots scripts/latexize.sh -tar -zcvf dagstuhloid.tgz dagstuhloid.pdf *.csv *plots/xpresults_all.png rnk_*.txt *plots/fight_all.png.cp.txt pie*.png agpi*.png +tar -zcvf dagstuhloid.tgz dagstuhloid.pdf *.csv *plots/xpresults_all.png rnk_*.txt *plots/fight_all.png.cp.txt *pie*.png diff --git a/scripts/tex/end.tex b/scripts/tex/end.tex index f3738a72e..41175595a 100644 --- a/scripts/tex/end.tex +++ b/scripts/tex/end.tex @@ -3,15 +3,11 @@ \section{Global pies} \subsection{Including all} -\includegraphics[width=.48\textwidth]{pie1.png} -\includegraphics[width=.48\textwidth]{pie2.png} -\includegraphics[width=.48\textwidth]{pie4.png} -\includegraphics[width=.48\textwidth]{pie8.png} +\includegraphics[width=.48\textwidth]{pierob1.png} +\includegraphics[width=.48\textwidth]{pierob3.png} \subsection{No NgIohTuned, no NGDSRW, only one per category} -\includegraphics[width=.48\textwidth]{pie1.png} -\includegraphics[width=.48\textwidth]{pie2.png} -\includegraphics[width=.48\textwidth]{pie4.png} -\includegraphics[width=.48\textwidth]{pie8.png} +\includegraphics[width=.48\textwidth]{pierob1.png} +\includegraphics[width=.48\textwidth]{pierob3.png} \section{Comparison with baseline} From 68dac71adf788faafe1e703a88d8f904d61a3d9f Mon Sep 17 00:00:00 2001 From: Olivier Teytaud Date: Tue, 19 Nov 2024 10:50:01 +0200 Subject: [PATCH 7/8] adapt --- nevergrad/benchmark/experiments.py | 98 +++++++----- nevergrad/benchmark/gymexperiments.py | 210 ++++++++++++++++--------- nevergrad/benchmark/plotting.py | 6 +- nevergrad/benchmark/xpbase.py | 12 +- nevergrad/optimization/optimizerlib.py | 5 +- scripts/pies_multirank.sh | 161 +++++++++++++++++++ 6 files changed, 376 insertions(+), 116 deletions(-) create mode 100755 scripts/pies_multirank.sh diff --git a/nevergrad/benchmark/experiments.py b/nevergrad/benchmark/experiments.py index f97464abf..0e665bf3a 100644 --- a/nevergrad/benchmark/experiments.py +++ b/nevergrad/benchmark/experiments.py @@ -54,45 +54,69 @@ # list_optims = ["QOTPDE", "LQOTPDE", "LQODE"] # list_optims = ["SPQODE", "SQOPSO", "DiagonalCMA"] def refactor_optims(x: tp.List[tp.Any]) -> tp.List[tp.Any]: # type: ignore - #return ["SMAC3"] - #return ["AXP"] - #return [ "RandomSearch","Cobyla","NgIohTuned","PymooBIPOP","CMA","PSO","SQOPSO","DE","DiscreteLenglerOnePlusOne","DiscreteOnePlusOne","OnePlusOne","DSproba","MetaModel","LognormalDiscreteOnePlusOne","CauchyRandomSearch","RandomScaleRandomSearchPlusMiddlePoint","HullAvgMetaTuneRecentering","HyperOpt","NGDSRW"] - return [np.random.choice([ "RandomSearch","AXP","Cobyla","PCABO","SMAC3","NgIohTuned","PymooBIPOP","CMA","PSO","SQOPSO","DE","DiscreteLenglerOnePlusOne","DiscreteOnePlusOne","OnePlusOne","DSproba","MetaModel","LognormalDiscreteOnePlusOne","CauchyRandomSearch","RandomScaleRandomSearchPlusMiddlePoint","HullAvgMetaTuneRecentering","HyperOpt","NGDSRW"])] - return ["RandomSearch"] - return ["HyperOpt"] - return ["CauchyRandomSearch"] - return ["RandomScaleRandomSearchPlusMiddlePoint"] - return ["HullAvgMetaTuneRecentering"] - return list( - np.random.choice( - [ - "RandomSearch", - "AXP", - "LognormalDiscreteOnePlusOne", - "CMA", - "Cobyla", - "PCABO", - "SMAC3", - "NgIohTuned", - "NGDSRW", - "PymooBIPOP", - "CMA", - "PSO", - "SQOPSO", - "DE", - "DiscreteLenglerOnePlusOne", - "DiscreteOnePlusOne", - "OnePlusOne", - "DSproba", - "MetaModel", - ], - 1, - ) - ) # return [ - # np.random.choice(["RandomSearch", "AX", "CMA", "Cobyla", "PCABO", "SMAC3", "NgIohTuned", "NGDSRW"]) + # np.random.choice( + # [ + # "RandomSearch", + # "AXP", + # "Cobyla", + # "PCABO", + # "SMAC3", + # "NgIohTuned", + # "PymooBIPOP", + # "CMA", + # "PSO", + # "SQOPSO", + # "DE", + # "DiscreteLenglerOnePlusOne", + # "DiscreteOnePlusOne", + # "OnePlusOne", + # "DSproba", + # "MetaModel", + # "LognormalDiscreteOnePlusOne", + # "CauchyRandomSearch", + # "RandomScaleRandomSearchPlusMiddlePoint", + # "HullAvgMetaTuneRecentering", + # "HyperOpt", + # "NGDSRW", + # ] + # ) # ] - # return ["NgIoh4", "NgIoh21", "NgIohTuned", "NgIohLn", "NgIohRS", "LBFGSB"] + # return ["RandomSearch"] + # return ["HyperOpt"] + # return ["CauchyRandomSearch"] + # return ["RandomScaleRandomSearchPlusMiddlePoint"] + # return ["HullAvgMetaTuneRecentering"] + # return list( + # np.random.choice( + # [ + # "RandomSearch", + # "AXP", + # "LognormalDiscreteOnePlusOne", + # "CMA", + # "Cobyla", + # "PCABO", + # "SMAC3", + # "NgIohTuned", + # "NGDSRW", + # "PymooBIPOP", + # "CMA", + # "PSO", + # "SQOPSO", + # "DE", + # "DiscreteLenglerOnePlusOne", + # "DiscreteOnePlusOne", + # "OnePlusOne", + # "DSproba", + # "MetaModel", + # ], + # 1, + # ) + # ) + # # return [ + # # np.random.choice(["RandomSearch", "AX", "CMA", "Cobyla", "PCABO", "SMAC3", "NgIohTuned", "NGDSRW"]) + # # ] + # # return ["NgIoh4", "NgIoh21", "NgIohTuned", "NgIohLn", "NgIohRS", "LBFGSB"] if False: # np.random.randn() < 0.0: return list( np.random.choice( diff --git a/nevergrad/benchmark/gymexperiments.py b/nevergrad/benchmark/gymexperiments.py index 4eca2a8cb..9f2578187 100644 --- a/nevergrad/benchmark/gymexperiments.py +++ b/nevergrad/benchmark/gymexperiments.py @@ -3,7 +3,7 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -import numpy as np +# import numpy as np import os import typing as tp from nevergrad.functions import gym as nevergrad_gym @@ -23,29 +23,15 @@ def gym_problem_modifier(specific_problem): def gym_optimizer_modifier(optims): - return [np.random.choice([ "RandomSearch","AXP","Cobyla","PCABO","SMAC3","NgIohTuned","PymooBIPOP","CMA","PSO","SQOPSO","DE","DiscreteLenglerOnePlusOne","DiscreteOnePlusOne","OnePlusOne","DSproba","MetaModel","LognormalDiscreteOnePlusOne","CauchyRandomSearch","RandomScaleRandomSearchPlusMiddlePoint","HullAvgMetaTuneRecentering","HyperOpt","NGDSRW"])] - return [ "RandomSearch","Cobyla","NgIohTuned","PymooBIPOP","CMA","PSO","SQOPSO","DE","DiscreteLenglerOnePlusOne","DiscreteOnePlusOne","OnePlusOne","DSproba","MetaModel","LognormalDiscreteOnePlusOne","CauchyRandomSearch","RandomScaleRandomSearchPlusMiddlePoint","HullAvgMetaTuneRecentering","HyperOpt","NGDSRW"] - return [ "RandomSearch","AXP","Cobyla","PCABO","SMAC3","NgIohTuned","PymooBIPOP","CMA","PSO","SQOPSO","DE","DiscreteLenglerOnePlusOne","DiscreteOnePlusOne","OnePlusOne","DSproba","MetaModel","LognormalDiscreteOnePlusOne","CauchyRandomSearch","RandomScaleRandomSearchPlusMiddlePoint","HullAvgMetaTuneRecentering","HyperOpt","NGDSRW"] - return ["RandomSearch"] - return ["HyperOpt"] - return ["CauchyRandomSearch"] - return ["RandomScaleRandomSearchPlusMiddlePoint"] - return ["HullAvgMetaTuneRecentering"] - return ["SMAC3"] - return ["AXP"] - # return list( + # return [ # np.random.choice( # [ # "RandomSearch", # "AXP", - # "LognormalDiscreteOnePlusOne", - # "AX", - # "CMA", # "Cobyla", # "PCABO", # "SMAC3", # "NgIohTuned", - # "NGDSRW", # "PymooBIPOP", # "CMA", # "PSO", @@ -56,63 +42,147 @@ def gym_optimizer_modifier(optims): # "OnePlusOne", # "DSproba", # "MetaModel", - # ], - # 1, - # ) - # ) - # return list( - # np.random.choice( - # [ - # "RandomSearch", - # "AX", - # "CMA", - # "Cobyla", - # "PCABO", - # "SMAC3", - # "NgIohTuned", - # "NGDSRW", - # "PymooBIPOP", - # "CMA", - # "PSO", - # "SQOPSO", - # "DE", - # "DiscreteLenglerOnePlusOne", - # "DiscreteOnePlusOne", - # "OnePlusOne", - # "DSproba", - # "MetaModel", - # ], - # 3, - # ) - # ) - # return list( - # np.random.choice( - # [ - # "RandomSearch", - # "AXP", # "LognormalDiscreteOnePlusOne", - # "AX", - # "CMA", - # "Cobyla", - # "PCABO", - # "SMAC3", - # "NgIohTuned", + # "CauchyRandomSearch", + # "RandomScaleRandomSearchPlusMiddlePoint", + # "HullAvgMetaTuneRecentering", + # "HyperOpt", # "NGDSRW", - # "PymooBIPOP", - # "CMA", - # "PSO", - # "SQOPSO", - # "DE", - # "DiscreteLenglerOnePlusOne", - # "DiscreteOnePlusOne", - # "OnePlusOne", - # "DSproba", - # "MetaModel", - # ], - # 1, + # ] # ) - # ) - print(optims) + # ] + # return [ + # "RandomSearch", + # "Cobyla", + # "NgIohTuned", + # "PymooBIPOP", + # "CMA", + # "PSO", + # "SQOPSO", + # "DE", + # "DiscreteLenglerOnePlusOne", + # "DiscreteOnePlusOne", + # "OnePlusOne", + # "DSproba", + # "MetaModel", + # "LognormalDiscreteOnePlusOne", + # "CauchyRandomSearch", + # "RandomScaleRandomSearchPlusMiddlePoint", + # "HullAvgMetaTuneRecentering", + # "HyperOpt", + # "NGDSRW", + # ] + # return [ + # "RandomSearch", + # "AXP", + # "Cobyla", + # "PCABO", + # "SMAC3", + # "NgIohTuned", + # "PymooBIPOP", + # "CMA", + # "PSO", + # "SQOPSO", + # "DE", + # "DiscreteLenglerOnePlusOne", + # "DiscreteOnePlusOne", + # "OnePlusOne", + # "DSproba", + # "MetaModel", + # "LognormalDiscreteOnePlusOne", + # "CauchyRandomSearch", + # "RandomScaleRandomSearchPlusMiddlePoint", + # "HullAvgMetaTuneRecentering", + # "HyperOpt", + # "NGDSRW", + # ] + # return ["RandomSearch"] + # return ["HyperOpt"] + # return ["CauchyRandomSearch"] + # return ["RandomScaleRandomSearchPlusMiddlePoint"] + # return ["HullAvgMetaTuneRecentering"] + # return ["SMAC3"] + # return ["AXP"] + # # return list( + # # np.random.choice( + # # [ + # # "RandomSearch", + # # "AXP", + # # "LognormalDiscreteOnePlusOne", + # # "AX", + # # "CMA", + # # "Cobyla", + # # "PCABO", + # # "SMAC3", + # # "NgIohTuned", + # # "NGDSRW", + # # "PymooBIPOP", + # # "CMA", + # # "PSO", + # # "SQOPSO", + # # "DE", + # # "DiscreteLenglerOnePlusOne", + # # "DiscreteOnePlusOne", + # # "OnePlusOne", + # # "DSproba", + # # "MetaModel", + # # ], + # # 1, + # # ) + # # ) + # # return list( + # # np.random.choice( + # # [ + # # "RandomSearch", + # # "AX", + # # "CMA", + # # "Cobyla", + # # "PCABO", + # # "SMAC3", + # # "NgIohTuned", + # # "NGDSRW", + # # "PymooBIPOP", + # # "CMA", + # # "PSO", + # # "SQOPSO", + # # "DE", + # # "DiscreteLenglerOnePlusOne", + # # "DiscreteOnePlusOne", + # # "OnePlusOne", + # # "DSproba", + # # "MetaModel", + # # ], + # # 3, + # # ) + # # ) + # # return list( + # # np.random.choice( + # # [ + # # "RandomSearch", + # # "AXP", + # # "LognormalDiscreteOnePlusOne", + # # "AX", + # # "CMA", + # # "Cobyla", + # # "PCABO", + # # "SMAC3", + # # "NgIohTuned", + # # "NGDSRW", + # # "PymooBIPOP", + # # "CMA", + # # "PSO", + # # "SQOPSO", + # # "DE", + # # "DiscreteLenglerOnePlusOne", + # # "DiscreteOnePlusOne", + # # "OnePlusOne", + # # "DSproba", + # # "MetaModel", + # # ], + # # 1, + # # ) + # # ) + # print(optims) if os.environ.get("GYM_OPTIMIZER") is not None: optimizer_string = os.environ.get("GYM_OPTIMIZER") print(f"Considering optimizers with {optimizer_string} in their name.") diff --git a/nevergrad/benchmark/plotting.py b/nevergrad/benchmark/plotting.py index 46d3f04c5..a973d78f5 100644 --- a/nevergrad/benchmark/plotting.py +++ b/nevergrad/benchmark/plotting.py @@ -448,7 +448,11 @@ def create_plots( data = XpPlotter.make_data(df, normalized_loss=True) for pure_only in [False, True]: xpplotter = XpPlotter( - data, title=os.path.basename(output_folder), name_style=name_style, xaxis=xpaxis, pure_only=pure_only + data, + title=os.path.basename(output_folder), + name_style=name_style, + xaxis=xpaxis, + pure_only=pure_only, ) except Exception as e: lower = 0 diff --git a/nevergrad/benchmark/xpbase.py b/nevergrad/benchmark/xpbase.py index ec6ce405e..99d07e3cd 100644 --- a/nevergrad/benchmark/xpbase.py +++ b/nevergrad/benchmark/xpbase.py @@ -291,11 +291,11 @@ def _run_with_error(self, callbacks: tp.Optional[tp.Dict[str, obase._OptimCallBa try: # call the actual Optimizer.minimize method because overloaded versions could alter the worklflow # and provide unfair comparisons (especially for parallelized settings) - import signal - def handler(signum, frame): - raise Exception("Time exceeded!") - signal.signal(signal.SIGALRM, handler) - signal.alarm(int(2.8 * 3600 * 24)) + # import signal + # def handler(signum, frame): + # raise Exception("Time exceeded!") + # signal.signal(signal.SIGALRM, handler) + # signal.alarm(int(2.8 * 3600 * 24)) if self.suggestions is not None: for s in self.suggestions: self._optimizer.suggest(s) @@ -307,7 +307,7 @@ def handler(signum, frame): constraint_violation=self.constraint_violation, max_time=3600 * 24 * 2.5, ) - signal.alarm(0) + # signal.alarm(0) except Exception as e: # pylint: disable=broad-except self._log_results(pfunc, t0, self._optimizer.num_ask) raise e diff --git a/nevergrad/optimization/optimizerlib.py b/nevergrad/optimization/optimizerlib.py index 0be185989..a6f9324a3 100644 --- a/nevergrad/optimization/optimizerlib.py +++ b/nevergrad/optimization/optimizerlib.py @@ -1188,6 +1188,7 @@ def _internal_ask_candidate(self) -> p.Parameter: def invsig(x): def p(x): return np.clip(x, 1e-15, 1.0 - 1e-15) + return np.log(p(x) / (1 - p(x))) if len(self._trials) == 0: @@ -3977,8 +3978,8 @@ def _select_optimizer_cls(self) -> base.OptCls: return optCls -#@registry.register -#class MetaBO(NGOptBase): +# @registry.register +# class MetaBO(NGOptBase): # """Nevergrad optimizer by competence map. You might modify this one for designing your own competence map.""" # # def _select_optimizer_cls(self) -> base.OptCls: diff --git a/scripts/pies_multirank.sh b/scripts/pies_multirank.sh new file mode 100755 index 000000000..e044df37d --- /dev/null +++ b/scripts/pies_multirank.sh @@ -0,0 +1,161 @@ +#!/bin/bash + +#python -m pip install git+https://github.com/maroofi/csvtool.git + +touch pie.png +mkdir -p poubelle +mv *pie*.png poubelle/ + +# Precompute ratios +for f in *.csv +do +pbname=`echo $f | sed 's/\.csv//g'` +touch log_ratio_${pbname}.log +rm log_ratio_${pbname}.log +scripts/get_average_budget_per_dim_of_csv.sh ${pbname} | awk '{print $2}' > log_ratio_${pbname}.log +done + +for robust in no yes +do +( +echo 'import matplotlib' +echo 'import matplotlib.pyplot as plt' + +#for k in rnk*.txt +#do +#echo -n "# $k :" +#grep ':' $k | sed 's/.*://g' | sed 's/ (.*//g' | egrep '[a-zA-Z0-9]' | sort | uniq | wc -l +# +#done + +for targetratio in . _1 0 1 2 +do +echo "# Working on targetratio $targetratio" +echo "targetratio=\"`echo $targetratio | sed 's/_/\-/g'`\"" +echo "targetratio_=\"`echo $targetratio | sed 's/\.//g' `\"" +# Agregated or not ? +for ag in yes no +do +# How many best are kept ? +for num in 1 3 +do +echo "num=$num" +if [ "$robust" == yes ]; then +echo "numt=$( ls */*.cp.txt | wc -l )" +else +echo "numt=$( ls rnk* | wc -l )" +fi +echo "labels = []" +echo "numbers = []" +echo "realnumbers = []" +echo "def ref(x):" +echo ' return x.replace("MiddlePoint","Mid").replace("Discrete", "Disc").replace("OnePlusOne", "(1+1)").replace("RandomSearch","RS").replace("Random", "Rnd").replace("OnePlusOne","(1+1)").replace("Recentering","Cent")' +echo "# $num best" + +# Determinining the list of files +if [ "$robust" == yes ]; then + echo "rob='rob'" + list="$( ls */*.png.cp.txt )" +else + echo "rob=''" + list="$( ls rnk*.txt | grep -v pure )" +fi + + +fulllistalgos="RandomSearch AXP Cobyla PCABO SMAC3 NgIohTuned PymooBIPOP CMA PSO SQOPSO DE DiscreteLenglerOnePlusOne DiscreteOnePlusOne OnePlusOne DSproba MetaModel LognormalDiscreteOnePlusOne CauchyRandomSearch RandomScaleRandomSearchPlusMiddlePoint HullAvgMetaTuneRecentering HyperOpt NGDSRW" +listalgos="$fulllistalgos" +if [ "$ag" == yes ]; then +echo "# pruning!" +listalgos2="" +for l in $listalgos +do + r=$( echo " $l " | egrep -v ' NgIohTuned | NGDSRW | AX | SMAC3 | LognormalDiscreteOnePlusOne | PCABO | PSO | PymooBIPOP | CMA | RandomSearch | RandomScaleRandomSearchPlusMiddlePoint | HullAvgMetaTuneRecentering | HyperOpt ' ) + listalgos2="$listalgos2 $r" +done +listalgos="$listalgos2" +fi + +echo "# ag=$ag listalgos=$( echo $listalgos )" + + +for a in $listalgos +do +# for k in $list +# do +# pbname=`echo $k | sed 's/_plots.*//g' | grep -v pure | sed 's/rnk__//g' ` +# ratio=$( scripts/get_average_budget_per_dim_of_csv.sh ${pbname} | awk '{print $2}' | sed 's/\-/_/g' ) +# if [ "$ratio" == "$targetratio" ] | [ "$targetratio" == . ] ; then +# echo "# $pbname $ratio $targetraio selected" +# else +# echo "# $pbname $ratio $targetraio unselected" +# fi +# done +echo -n "# $a is in the $num best in this number of problems:" +number=$( +for k in $list +do + + pbname=`echo $k | sed 's/_plots.*//g' | grep -v pure | sed 's/rnk__//g' |sed 's/.*\///g'` + #echo "# Working on pbname=${pbname} by k=${k} for rob=$rob num=$num" + #touch $( echo "Gotouch Working on pbname=${pbname} by k=${k} for rob=$rob num=$num" | sed 's/\//_/g' | sed 's/ /_/g' ) + #ratio=$( scripts/get_average_budget_per_dim_of_csv.sh ${pbname} | awk '{print $2}' | sed 's/\-/_/g' ) + ratio=`cat log_ratio_${pbname}.log | sed 's/\-/_/g'` + if [ "$ratio" == "$targetratio" ] || [ "$targetratio" == . ] ; then + touch sentinel__${pbname}__${ratio}_${targetratio}_yes.log + if [ "$ag" == yes ]; then + grep ':' $k | egrep ' RandomSearch | AXP | Cobyla | PCABO | SMAC3 | NgIohTuned | PymooBIPOP | CMA | PSO | SQOPSO | DE | DiscreteLenglerOnePlusOne | DiscreteOnePlusOne | OnePlusOne | DSproba | MetaModel | LognormalDiscreteOnePlusOne | CauchyRandomSearch | RandomScaleRandomSearchPlusMiddlePoint | HullAvgMetaTuneRecentering | HyperOpt | NGDSRW ' | grep -v ' AX ' | sed 's/.*://g' | sed 's/ (.*//g' | egrep '[a-zA-Z0-9]' | egrep -v ' NgIohTuned$| NGDSRW$| AX$| SMAC3$| LognormalDiscreteOnePlusOne$| PCABO$| PSO$| PymooBIPOP$| CMA$| RandomSearch$| RandomScaleRandomSearchPlusMiddlePoint$| HullAvgMetaTuneRecentering$| HyperOpt$' | tee check${targetratio}_${ag}_${num}_for${pbname}.log | head -n $num | grep "^ $a$" + else + grep ':' $k | egrep ' RandomSearch | AXP | Cobyla | PCABO | SMAC3 | NgIohTuned | PymooBIPOP | CMA | PSO | SQOPSO | DE | DiscreteLenglerOnePlusOne | DiscreteOnePlusOne | OnePlusOne | DSproba | MetaModel | LognormalDiscreteOnePlusOne | CauchyRandomSearch | RandomScaleRandomSearchPlusMiddlePoint | HullAvgMetaTuneRecentering | HyperOpt | NGDSRW ' | grep -v ' AX ' | sed 's/.*://g' | sed 's/ (.*//g' | egrep '[a-zA-Z0-9]' |tee check${targetratio}_${ag}_${num}_for${pbname}.log | head -n $num | grep "^ $a$" + fi + else + touch sentinel__${pbname}__${ratio}_${targetratio}_no.log + fi +done | wc -l ) +echo $number +if [ "$ag" == yes ]; then +echo "# AG yes $a" +echo "labels += [\" $a \" + '(' + str( $number ) + ')' ];realnumbers += [ $number ] ; numbers += [ $number + .1 ]" +else +echo "# AG no $a" +echo "labels += [ref(\" $a \") + '(' + str( $number ) + ')' ];realnumbers += [ $number ] ; numbers += [ $number + .1 ]" +fi +done +echo 'plt.clf()' +echo 'def get_cmap(n, name="hsv"):' +echo ' return plt.cm.get_cmap(name, n)' +#echo ' return plt.cm.get_cmap(name, n)' +echo 'colors=get_cmap(len(labels))' +echo "plt.bar(labels, numbers, label=labels, color=[colors(i) for i in range(len(numbers))])" +echo 'plt.xticks(rotation=90)' +echo "title=f'How many times in the {num} best ? out of {sum(realnumbers)}'" +echo "plt.title(title)" +echo "plt.tight_layout()" +if [ "$ag" == yes ]; then +echo "plt.savefig(f'ag{targetratio_}piebar{rob}{num}.png')" +else +echo "plt.savefig(f'{targetratio_}piebar{rob}{num}.png')" +fi +echo 'plt.clf()' +echo "plt.pie(numbers, labels=labels)" +echo "title=f'How many times in the {num} best ? out of {sum(realnumbers)}'" +echo "plt.title(title)" +echo "plt.tight_layout()" +if [ "$ag" == yes ]; then +echo "plt.savefig(f'ag{targetratio_}pie{rob}{num}.png')" +else +echo "plt.savefig(f'{targetratio_}pie{rob}{num}.png')" +fi +done +done +done +) > plotpie.py + +python plotpie.py +done +#sed -i 's/label.* AX .*//g' plotpie.py +#sed -i 's/label.* PSO .*//g' plotpie.py +#sed -i 's/label.* PymooBIPOP .*//g' plotpie.py +#sed -i 's/label.* CMA .*//g' plotpie.py +#sed -i 's/pie{num}/agpie{num}/g' plotpie.py +# +#python plotpie.py From 8b5a66fdd6f92af02a4b5c1cfef4e6d57d8c010d Mon Sep 17 00:00:00 2001 From: Teytaud Date: Tue, 19 Nov 2024 10:42:14 +0100 Subject: [PATCH 8/8] Update optimizerlib.py --- nevergrad/optimization/optimizerlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nevergrad/optimization/optimizerlib.py b/nevergrad/optimization/optimizerlib.py index c9cc37bee..f23fd2d04 100644 --- a/nevergrad/optimization/optimizerlib.py +++ b/nevergrad/optimization/optimizerlib.py @@ -1173,7 +1173,7 @@ def __init__( objectives={"result": ObjectiveProperties(minimize=True)}, # parameter_constraints=["x + y <= 2.0"], # Optional. ) - self._trials = [] + self._trials = [] # type: ignore def _internal_ask_candidate(self) -> p.Parameter: def invsig(x):