Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

metamodels, one more version #1643

Merged
merged 48 commits into from
Oct 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
48 commits
Select commit Hold shift + click to select a range
9f04943
fix
teytaud May 29, 2024
5c792a7
fix
teytaud Jun 6, 2024
577a323
multiceviche
teytaud Jun 7, 2024
4690f71
fix
teytaud Jun 11, 2024
5d7d989
nocevicheincircleci
teytaud Jun 12, 2024
27f21f5
manyfixed
teytaud Jun 12, 2024
babdbda
manyfixed
teytaud Jun 12, 2024
ffabba8
ola
teytaud Jun 12, 2024
7415670
po
teytaud Jun 12, 2024
d6ad142
po
teytaud Jun 14, 2024
feb6b76
ceviche
teytaud Jul 4, 2024
71e80c8
newceviche
teytaud Jul 4, 2024
e4355c7
ok_firstversion
teytaud Jul 5, 2024
c8d537d
plotceviche
teytaud Jul 5, 2024
85a7257
pop
teytaud Jul 9, 2024
dc4aee4
Merge branch 'main' into newceviche
teytaud Jul 9, 2024
6ae8636
clean
teytaud Jul 9, 2024
5582a95
newce
teytaud Jul 16, 2024
db5c2dd
cev
teytaud Jul 16, 2024
5832a28
pouet
teytaud Jul 17, 2024
f9859de
improve
teytaud Aug 13, 2024
623ab72
fix
teytaud Aug 17, 2024
9b28889
tryfix
teytaud Aug 19, 2024
ee9c7c7
Merge branch 'main' into newceviche
teytaud Aug 19, 2024
882de10
Update plotting.py
teytaud Aug 19, 2024
b0f3aa1
black_and_plotting
teytaud Aug 28, 2024
e9fa8eb
black_and_all
teytaud Sep 3, 2024
1a63337
ref
teytaud Sep 3, 2024
49393bd
Add surrogate models on Discrete Lognormal (#1627) (#1628)
teytaud Sep 3, 2024
7fe441f
fix
teytaud Sep 3, 2024
06159d5
black
teytaud Sep 9, 2024
93d1b80
newversion
teytaud Sep 10, 2024
d7ba860
Merge branch 'main' into newceviche
teytaud Sep 10, 2024
42936da
metamodel1
teytaud Sep 10, 2024
0079ad1
waow
teytaud Sep 10, 2024
e9e5075
po
teytaud Sep 10, 2024
9fc063e
fix
teytaud Sep 12, 2024
df6b45f
Merge branch 'main' into mm
teytaud Sep 12, 2024
0a51d30
mm
teytaud Sep 12, 2024
82548bb
somethingweird
teytaud Sep 13, 2024
3b14b70
black
teytaud Sep 19, 2024
c65ae67
prog
teytaud Sep 23, 2024
42b684e
Merge branch 'main' into newmm2
teytaud Sep 23, 2024
ee2deec
Merge branch 'main' into newmm2
teytaud Sep 23, 2024
1e1a23d
impro
teytaud Sep 23, 2024
d70c4fe
fix
teytaud Oct 1, 2024
6b7560a
po
teytaud Oct 3, 2024
b551553
Merge branch 'main' into newmm3
teytaud Oct 3, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions examples/convert_ceviche_npy.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
import glob
import numpy as np
import matplotlib.pyplot as plt
import os.path

def rename(s):
return s.replace("pb0", "pb0bend").replace("pb1","pb1beamsplitter").replace("pb2", "pb2modeconverter")

num = len(glob.glob("*.npy"))


for idx, f in enumerate(glob.glob("*iel*.npy")):
printing = False
if int(np.sqrt(idx)) < int(np.sqrt(idx+1)):
print(idx, " / ", num, " : ", f)
printing = True
if (("0.000" not in f and "pb0" in f) or ("0.00" not in f) or ("pb3" in f and "0.0" not in f)) and "WS" not in f:
if printing:
print(" ... skipping ")
continue
data = np.load(f)
#print('Shape:',data.shape)
if len(data.shape) <3:
continue
assert data.shape[0] < 3, f"{f} leads to an issue!"
assert data.shape[1] == 1
assert len(data.shape) == 4
if data.shape[0]==2:
target_name = rename(f) + "_1st_wavelength.png"
target_name2 = rename(f) + "_2nd_wavelength.png"
if not os.path.isfile(target_name):
field = np.abs(data[0, 0, :, :])**2
plt.imshow(field.T, cmap='viridis')
plt.savefig(target_name)
field = np.abs(data[1, 0, :, :])**2
plt.imshow(field.T, cmap='viridis')
plt.savefig(target_name2)
else:
target_name = rename(f) + ".png"
if not os.path.isfile(target_name):
field = np.abs(data[0, 0, :, :])**2
plt.imshow(field.T, cmap='viridis')
plt.savefig(target_name)

108 changes: 80 additions & 28 deletions nevergrad/benchmark/experiments.py
Original file line number Diff line number Diff line change
Expand Up @@ -3452,6 +3452,7 @@ def multi_ceviche(
seed: tp.Optional[int] = None,
c0: bool = False,
precompute: bool = False,
warmstart: bool = False,
) -> tp.Iterator[Experiment]:
"""Categories when running with c0:
BFGScheat works on the continuous problem, with continuous domain, with continuous test.
Expand Down Expand Up @@ -3505,19 +3506,45 @@ def multi_ceviche(
"ZetaSmoothDiscreteLognormalOnePlusOne",
"SuperSmoothDiscreteLognormalOnePlusOne",
]
algos = [a for a in algos if a in list(ng.optimizers.registry.keys())]
# if np.random.choice([True,False]):
# algos = refactor_optims(algos)
# algo = np.random.choice(algos)
assert not (precompute and not warmstart)
if not precompute:
algos = ["RF1MetaModelLogNormal", "Neural1MetaModelLogNormal", "SVM1MetaModelLogNormal", "CMAL"]
else:
algos = ["UltraSmoothDiscreteLognormalOnePlusOne", "DiscreteLenglerOnePlusOne", "CMA", "CMAL"]
algos = ["CMALS", "CMALYS", "CMALL"]
algos = ["CLengler", "CMALS", "CMALYS", "CMALL", "CMAL"]
algos = ["CMASL2", "CMASL3"]
algos = [
"DiagonalCMA",
"CMAL3",
"CMA",
"CLengler",
"CMALL",
"CMALYS",
"CMALS" "DiscreteLenglerOnePlusOne",
"CMASL3",
"CMASL2",
"DSproba",
]
algos += [
"LognormalDiscreteOnePlusOne",
"CMA",
"DiscreteLenglerOnePlusOne",
"SmoothDiscreteLognormalOnePlusOne",
"SuperSmoothDiscreteLognormalOnePlusOne",
"AnisotropicAdaptiveDiscreteOnePlusOne",
"RFMetaModelLogNormal",
"NeuralMetaModelLogNormal",
"RFMetaModelLogNormal",
"NeuralMetaModelLogNormal",
"SVMMetaModelLogNormal",
# "RFMetaModelLogNormal",
# "NeuralMetaModelLogNormal",
# "RFMetaModelLogNormal",
# "NeuralMetaModelLogNormal",
# "SVMMetaModelLogNormal",
"Neural1MetaModelE",
"SVM1MetaModelE",
"Quad1MetaModelE",
"RF1MetaModelE",
"UltraSmoothDiscreteLognormalOnePlusOne",
"VoronoiDE",
"UltraSmoothDiscreteLognormalOnePlusOne",
Expand All @@ -3526,19 +3553,23 @@ def multi_ceviche(
"Neural1MetaModelLogNormal",
"SVM1MetaModelLogNormal",
"DSproba",
"ImageMetaModelE",
"ImageMetaModelOnePlusOne",
"ImageMetaModelDiagonalCMA",
"ImageMetaModelLengler",
"ImageMetaModelLogNormal",
]
# if np.random.choice([True,False]):
# algos = refactor_optims(algos)
# algo = np.random.choice(algos)
if not precompute:
algos = ["RF1MetaModelLogNormal", "Neural1MetaModelLogNormal", "SVM1MetaModelLogNormal", "CMAL"]
else:
algos = ["UltraSmoothDiscreteLognormalOnePlusOne", "DiscreteLenglerOnePlusOne", "CMA", "CMAL"]
algos = ["CMALS", "CMALYS", "CMALL"]
algos = ["CLengler", "CMALS", "CMALYS", "CMALL", "CMAL"]
algos = ["CMASL2", "CMASL3"]

algos = [a for a in algos if a in list(ng.optimizers.registry.keys())]
for benchmark_type in [np.random.choice([0, 1, 2, 3])]: # [np.random.randint(4)]:
if warmstart:
try:
suggestion = np.load(f"bestnp{benchmark_type}.npy")
except Exception as e:
print(
"Be caereful! You need warmstart data for warmstarting :-) use scripts/plot_ceviche.sh."
)
raise e
shape = tuple([int(p) for p in list(photonics_ceviche(None, benchmark_type))]) # type: ignore
name = photonics_ceviche("name", benchmark_type) + str(shape) # type: ignore
# print(f"Shape = {shape} {type(shape)} {type(shape[0])}")
Expand Down Expand Up @@ -3591,10 +3622,13 @@ def epc(x):
# print(f"name = {name}")
import copy

def export_numpy(name, array): # type: ignore
def export_numpy(name, array, fields=None): # type: ignore
from PIL import Image
import numpy as np

if warmstart:
name = "WS" + name

freq = np.average(np.abs(array.flatten() - 0.5) < 0.35)
freq2 = np.average(np.abs(array.flatten() - 0.5) < 0.45)
x = (255 * (1 - array)).astype("uint8")
Expand All @@ -3603,6 +3637,9 @@ def export_numpy(name, array): # type: ignore
name,
[100 * np.average(np.abs(np.round(10 * x.flatten()) - i) < 0.1) for i in range(11)],
)
if fields is not None:
np.save(name + "fields.", fields)
np.save(name + "savedarray", array)

im = Image.fromarray(x)
im.convert("RGB").save(f"{name}_{freq}_{freq2}.png", mode="L")
Expand Down Expand Up @@ -3638,34 +3675,35 @@ def cv(x):
# replace=False,
# )
# ): # [int(np.random.choice([3, 20, 50, 90]))]: #[20, 50, 90]:
if np.random.rand() < 0.05 or precompute:
if (np.random.rand() < 0.05 or precompute) and not warmstart:
from scipy import optimize as scipyoptimize

x0 = np.random.rand(np.prod(shape)) # type: ignore
result = scipyoptimize.minimize(
fpc,
x0=x0,
method="L-BFGS-B",
tol=1e-9,
jac=True,
options={"maxiter": budget if not precompute else 102400},
bounds=[[0, 1] for _ in range(np.prod(shape))],
)
assert -1e-5 <= np.min(result.x.flatten())
assert np.max(result.x.flatten()) <= 1.0001
real_loss = epc(result.x.reshape(shape))
fake_loss = fpc(result.x.reshape(shape))[0]
fake_loss, _ignored_gradient = fpc(result.x.reshape(shape))
if not precompute:
print(f"\nLOGPB{benchmark_type} LBFGSB with_budget {budget} returns {real_loss}")
print(
f"\nLOGPB{benchmark_type} CheatingLBFGSB with_budget {budget} returns {fake_loss}"
)
initial_point = result.x.reshape(shape)
if real_loss < -0.95: # and np.random.rand() < 0.9:
if budget > 100000 or np.random.rand() < 0.05:
export_numpy(
f"pb{benchmark_type}_budget{budget if not precompute else 102400}_bfgs_{real_loss}_{fake_loss}",
result.x.reshape(shape),
)
if (c0 and np.random.choice([True, False, False, False] + ([False] * 20))) and not precompute:
if (c0 and np.random.choice([True, False, False, False])) and not precompute:
pen = np.random.choice([True, False, False] + ([False] * 20)) and not precompute
pre_optim = ng.optimizers.registry[optim]
if pen:
Expand All @@ -3686,6 +3724,7 @@ def cv(x):
seed=next(seedg),
constraint_violation=[cv], # type: ignore
penalize_violation_at_test=False,
suggestions=([suggestion] if warmstart else None),
)
else:
cheat = np.random.choice([False, True])
Expand All @@ -3701,8 +3740,7 @@ def cv(x):
def plot_pc(x):
fake_loss = photonics_ceviche(x, benchmark_type)
real_loss = photonics_ceviche(x, benchmark_type, discretize=True)
if real_loss < -0.95 or np.random.rand() < 0.2:
print("exporting")
if budget > 100000 or np.random.rand() < 0.05:
export_numpy(
f"pb{benchmark_type}_{optim}c0c_budget{budget}_{real_loss}_fl{fake_loss}",
x.reshape(shape),
Expand All @@ -3721,21 +3759,29 @@ def plot_pc(x):
c0func if not cheat else c0cfunc,
evaluation=eval_func if not cheat else plot_cheat_eval_func,
)
yield Experiment(sfunc, optim3, budget=budget, seed=next(seedg)) # type: ignore
yield Experiment(sfunc, optim3, budget=budget, seed=next(seedg), suggestions=([suggestion] if warmstart else None)) # type: ignore
else:

def plot_epc(x):
real_loss = photonics_ceviche(x, benchmark_type, discretize=True)
if real_loss < -0.95 or np.random.rand() < 0.2:
real_loss, fields = photonics_ceviche(
x, benchmark_type, discretize=True, wantfields=True
)
if budget > 100000 or np.random.rand() < 0.05:
export_numpy(
f"pb{benchmark_type}_{optim}_budget{budget}_{real_loss}", x.reshape(shape)
f"pb{benchmark_type}_{optim}_budget{budget}_{real_loss}",
x.reshape(shape),
fields,
)
return real_loss

plot_eval_func = ExperimentFunction(plot_epc, instrum2p)
pfunc = helpers.SpecialEvaluationExperiment(func, evaluation=plot_eval_func)
yield Experiment(
func if np.random.rand() < 0.0 else pfunc, optim, budget=budget, seed=next(seedg)
func if np.random.rand() < 0.0 else pfunc,
optim,
budget=budget,
seed=next(seedg),
suggestions=([suggestion] if warmstart else None),
) # Once in the discrete case.


Expand All @@ -3745,6 +3791,12 @@ def multi_ceviche_c0(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
return multi_ceviche(seed, c0=True) # means that we include c0 cases.


@registry.register
def multi_ceviche_c0_warmstart(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
"""Counterpart of multi_ceviche with continuous permittivities."""
return multi_ceviche(seed, c0=True, warmstart=True) # means that we include c0 cases.


@registry.register
def multi_ceviche_c0p(seed: tp.Optional[int] = None) -> tp.Iterator[Experiment]:
"""Counterpart of multi_ceviche with continuous permittivities and warmstart."""
Expand Down
14 changes: 9 additions & 5 deletions nevergrad/benchmark/plotting.py
Original file line number Diff line number Diff line change
Expand Up @@ -553,11 +553,15 @@ def ceviche_sota() -> tp.Dict[str, tp.Tuple[float, float]]:
# grep LOGPB *.out | sed 's/.*://g' | sort | uniq -c | grep with_budget | awk '{ data[$2,"_",$5] += $7; num[$2,"_",$5] += 1 } END { for (u in data) { print u, data[u]/num[u], num[u]} } ' | sort -n | grep '400 '
# Also obtained by examples/plot_ceviches.sh
# After log files have been created by sbatch examples/ceviche.sh
ceviche["waveguide-bend"] = (-0.590207, 1000000) # Budget 400
ceviche["beam-splitter"] = (-0.623696, 1000000)
ceviche["mode-converter"] = (-0.634207, 1000000)
ceviche["wdm"] = (-0.603663, 100000)

ceviche["waveguide-bend"] = (0.0681388, 1000000) # Budget 400
ceviche["beam-splitter"] = (0.496512, 1000000)
ceviche["mode-converter"] = (0.181592, 1000000)
ceviche["wdm"] = (0.982352, 100000)
# LOGPB0 409600 0.0681388
# LOGPB1 204800 0.496512
# LOGPB2 204800 0.181592
# LOGPB3 51200 0.982352
#
# LOGPB0_3200 -0.590207
# LOGPB1_3200 -0.623696
# LOGPB2_3200 -0.634207
Expand Down
5 changes: 5 additions & 0 deletions nevergrad/benchmark/xpbase.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,8 +161,10 @@ def __init__(
seed: tp.Optional[int] = None,
constraint_violation: tp.Optional[ngtp.ArrayLike] = None,
penalize_violation_at_test: bool = True,
suggestions: tp.Optional[ngtp.ArrayLike] = None,
) -> None:
self.penalize_violation_at_test = penalize_violation_at_test
self.suggestions = suggestions
assert isinstance(function, fbase.ExperimentFunction), (
"All experiment functions should " "derive from ng.functions.ExperimentFunction"
)
Expand Down Expand Up @@ -289,6 +291,9 @@ def _run_with_error(self, callbacks: tp.Optional[tp.Dict[str, obase._OptimCallBa
try:
# call the actual Optimizer.minimize method because overloaded versions could alter the worklflow
# and provide unfair comparisons (especially for parallelized settings)
if self.suggestions is not None:
for s in self.suggestions:
self._optimizer.suggest(s)
obase.Optimizer.minimize(
self._optimizer,
pfunc,
Expand Down
Loading