From e61c68008920498ef67bb1ccd8c3caa957c0201e Mon Sep 17 00:00:00 2001 From: RobertTLange Date: Sat, 8 Oct 2022 11:39:08 +0100 Subject: [PATCH 01/13] Simple GA change --- README.md | 2 +- evosax/problems/control_gym.py | 1 - evosax/strategies/simple_ga.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d440cb2..6ec672b 100755 --- a/README.md +++ b/README.md @@ -260,7 +260,7 @@ If you use `evosax` in your research, please cite it as follows: } ``` -We acknowledge financial support the [Google TRC](https://sites.research.google/trc/about/) and the Deutsche +We acknowledge financial support by the [Google TRC](https://sites.research.google/trc/about/) and the Deutsche Forschungsgemeinschaft (DFG, German Research Foundation) under Germany's Excellence Strategy - EXC 2002/1 ["Science of Intelligence"](https://www.scienceofintelligence.de/) - project number 390523135. ## Development 👷 diff --git a/evosax/problems/control_gym.py b/evosax/problems/control_gym.py index 323eaad..678d072 100644 --- a/evosax/problems/control_gym.py +++ b/evosax/problems/control_gym.py @@ -1,6 +1,5 @@ import jax import jax.numpy as jnp -from functools import partial from typing import Optional import chex diff --git a/evosax/strategies/simple_ga.py b/evosax/strategies/simple_ga.py index 7d7e7e5..e20af97 100755 --- a/evosax/strategies/simple_ga.py +++ b/evosax/strategies/simple_ga.py @@ -117,7 +117,7 @@ def tell_strategy( state.sigma, ) # Keep mean across stored archive around for evaluation protocol - mean = archive.mean(axis=0) + mean = archive[0] return state.replace( fitness=fitness, archive=archive, sigma=sigma, mean=mean ) From 38ef30ea7cb890e5f9cb679706c2719e8aac38d3 Mon Sep 17 00:00:00 2001 From: RobertTLange Date: Sat, 19 Nov 2022 09:52:49 +0100 Subject: [PATCH 02/13] Add GA algos --- .gitignore | 2 + README.md | 9 +- evosax/__init__.py | 37 ++-- evosax/experimental/decodings/decoder.py | 2 - evosax/experimental/decodings/hyper.py | 1 - evosax/experimental/decodings/random.py | 7 +- evosax/restarts/termination.py | 5 +- evosax/strategies/__init__.py | 14 +- evosax/strategies/cma_es.py | 8 +- evosax/strategies/esmc.py | 116 ++++++++++++ evosax/strategies/gesmr_ga.py | 160 +++++++++++++++++ evosax/strategies/pgpe.py | 29 +-- evosax/strategies/samr_ga.py | 99 ++++++++++ evosax/strategies/snes.py | 101 +++++++++++ evosax/strategies/xnes.py | 219 ++++++++++------------- evosax/utils/eigen_decomp.py | 3 +- evosax/utils/reshape_fitness.py | 33 +++- evosax/utils/reshape_params.py | 84 ++------- examples/01_classic_benchmark.ipynb | 6 +- tests/conftest.py | 6 +- tests/test_param_reshape.py | 13 +- 21 files changed, 709 insertions(+), 245 deletions(-) create mode 100644 evosax/strategies/esmc.py create mode 100644 evosax/strategies/gesmr_ga.py create mode 100644 evosax/strategies/samr_ga.py create mode 100644 evosax/strategies/snes.py diff --git a/.gitignore b/.gitignore index ddc38be..1aa7a32 100755 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +des.py +bbob.py # Standard ROB excludes .sync-config.cson .vim-arsync diff --git a/README.md b/README.md index 6ec672b..c841768 100755 --- a/README.md +++ b/README.md @@ -39,7 +39,8 @@ state.best_member, state.best_fitness | CMA-ES | [Hansen & Ostermeier (2001)](http://www.cmap.polytechnique.fr/~nikolaus.hansen/cmaartic.pdf) | [`CMA_ES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/cma_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | Simple Gaussian | [Rechenberg (1978)](https://link.springer.com/chapter/10.1007/978-3-642-81283-5_8) | [`SimpleES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/simple_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | Simple Genetic | [Such et al. (2017)](https://arxiv.org/abs/1712.06567) | [`SimpleGA`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/simple_ga.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) -| x-NES | [Wierstra et al. (2014)](https://www.jmlr.org/papers/volume15/wierstra14a/wierstra14a.pdf) | [`xNES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/xnes.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| XNES | [Wierstra et al. (2014)](https://www.jmlr.org/papers/volume15/wierstra14a/wierstra14a.pdf) | [`XNES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/xnes.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| SNES | [Wierstra et al. (2014)](https://www.jmlr.org/papers/volume15/wierstra14a/wierstra14a.pdf) | [`SNES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/sxnes.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | Particle Swarm Optimization | [Kennedy & Eberhart (1995)](https://ieeexplore.ieee.org/document/488968) | [`PSO`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/pso.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | Differential Evolution | [Storn & Price (1997)](https://www.metabolic-economics.de/pages/seminar_theoretische_biologie_2007/literatur/schaber/Storn1997JGlobOpt11.pdf) | [`DE`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/de.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | Persistent ES | [Vicol et al. (2021)](http://proceedings.mlr.press/v139/vicol21a.html) | [`PersistentES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/persistent_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/04_lrate_pes.ipynb) @@ -54,6 +55,12 @@ state.best_member, state.best_fitness | RmES | [Li & Zhang (2017)](https://ieeexplore.ieee.org/document/8080257) | [`RmES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/rm_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | GLD | [Golovin et al. (2019)](https://arxiv.org/pdf/1911.06317.pdf) | [`GLD`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/gld.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | Simulated Annealing | [Rasdi Rere et al. (2015)](https://www.sciencedirect.com/science/article/pii/S1877050915035759) | [`SimAnneal`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/sim_anneal.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| ESMC | [Merchant et al. (2021)](https://proceedings.mlr.press/v139/merchant21a.html) | [`ESMC`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/esmc.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| DES | [Lange et al. (2022)]() | [`DES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/des.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| SAMR-GA | [Clune et al. (2008)](https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1000187) | [`SAMR_GA`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/samr_ga.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| GESMR-GA | [Kumar et al. (2022)](https://arxiv.org/abs/2204.04817) | [`GESMR_GA`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/gesmr_ga.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) + + ## Installation ⏳ diff --git a/evosax/__init__.py b/evosax/__init__.py index 80d52e6..716691c 100755 --- a/evosax/__init__.py +++ b/evosax/__init__.py @@ -1,4 +1,4 @@ -from .strategy import Strategy +from .strategy import Strategy, EvoState, EvoParams from .strategies import ( SimpleGA, SimpleES, @@ -9,7 +9,6 @@ PGPE, PBT, PersistentES, - xNES, ARS, Sep_CMA_ES, BIPOP_CMA_ES, @@ -21,6 +20,12 @@ RmES, GLD, SimAnneal, + SNES, + xNES, + ESMC, + DES, + SAMR_GA, + GESMR_GA, ) from .utils import FitnessShaper, ParameterReshaper, ESLog from .networks import NetworkMapper @@ -37,7 +42,6 @@ "PGPE": PGPE, "PBT": PBT, "PersistentES": PersistentES, - "xNES": xNES, "ARS": ARS, "Sep_CMA_ES": Sep_CMA_ES, "BIPOP_CMA_ES": BIPOP_CMA_ES, @@ -49,9 +53,23 @@ "RmES": RmES, "GLD": GLD, "SimAnneal": SimAnneal, + "SNES": SNES, + "xNES": xNES, + "ESMC": ESMC, + "DES": DES, + "SAMR_GA": SAMR_GA, + "GESMR_GA": GESMR_GA, } __all__ = [ + "Strategies", + "EvoState", + "EvoParams", + "FitnessShaper", + "ParameterReshaper", + "ESLog", + "NetworkMapper", + "ProblemMapper", "Strategy", "SimpleGA", "SimpleES", @@ -62,7 +80,6 @@ "PGPE", "PBT", "PersistentES", - "xNES", "ARS", "Sep_CMA_ES", "BIPOP_CMA_ES", @@ -74,10 +91,10 @@ "RmES", "GLD", "SimAnneal", - "Strategies", - "FitnessShaper", - "ParameterReshaper", - "ESLog", - "NetworkMapper", - "ProblemMapper", + "SNES", + "xNES", + "ESMC", + "DES", + "SAMR_GA", + "GESMR_GA", ] diff --git a/evosax/experimental/decodings/decoder.py b/evosax/experimental/decodings/decoder.py index f23cb5c..9a15017 100644 --- a/evosax/experimental/decodings/decoder.py +++ b/evosax/experimental/decodings/decoder.py @@ -8,13 +8,11 @@ def __init__( self, num_encoding_dims: int, placeholder_params: Union[chex.ArrayTree, chex.Array], - identity: bool = False, n_devices: Optional[int] = None, ): self.num_encoding_dims = num_encoding_dims self.total_params = num_encoding_dims self.placeholder_params = placeholder_params - self.identity = identity if n_devices is None: self.n_devices = jax.local_device_count() else: diff --git a/evosax/experimental/decodings/hyper.py b/evosax/experimental/decodings/hyper.py index 08feccf..975dede 100644 --- a/evosax/experimental/decodings/hyper.py +++ b/evosax/experimental/decodings/hyper.py @@ -39,7 +39,6 @@ def __init__( super().__init__( hyper_reshaper.total_params, placeholder_params, - identity, n_devices, ) self.hyper_reshaper = hyper_reshaper diff --git a/evosax/experimental/decodings/random.py b/evosax/experimental/decodings/random.py index 439124a..46b100a 100644 --- a/evosax/experimental/decodings/random.py +++ b/evosax/experimental/decodings/random.py @@ -12,17 +12,14 @@ def __init__( placeholder_params: Union[chex.ArrayTree, chex.Array], rng: chex.PRNGKey = jax.random.PRNGKey(0), rademacher: bool = False, - identity: bool = False, n_devices: Optional[int] = None, ): """Random Projection Decoder (Gaussian/Rademacher random matrix).""" - super().__init__( - num_encoding_dims, placeholder_params, identity, n_devices - ) + super().__init__(num_encoding_dims, placeholder_params, n_devices) self.rademacher = rademacher # Instantiate base reshaper class self.base_reshaper = ParameterReshaper( - placeholder_params, identity, n_devices, verbose=False + placeholder_params, n_devices, verbose=False ) self.vmap_dict = self.base_reshaper.vmap_dict diff --git a/evosax/restarts/termination.py b/evosax/restarts/termination.py index 4a15edc..35c465c 100644 --- a/evosax/restarts/termination.py +++ b/evosax/restarts/termination.py @@ -36,7 +36,10 @@ def cma_criterion( dC = jnp.diag(state.strategy_state.C) # Note: Criterion requires full covariance matrix for decomposition! C, B, D = full_eigen_decomp( - state.strategy_state.C, state.strategy_state.B, state.strategy_state.D + state.strategy_state.C, + state.strategy_state.B, + state.strategy_state.D, + state.strategy_state.gen_counter, ) # Stop if std of normal distrib is smaller than tolx in all coordinates diff --git a/evosax/strategies/__init__.py b/evosax/strategies/__init__.py index d445fe1..64e5036 100755 --- a/evosax/strategies/__init__.py +++ b/evosax/strategies/__init__.py @@ -7,7 +7,6 @@ from .pgpe import PGPE from .pbt import PBT from .persistent_es import PersistentES -from .xnes import xNES from .ars import ARS from .sep_cma_es import Sep_CMA_ES from .bipop_cma_es import BIPOP_CMA_ES @@ -19,6 +18,12 @@ from .rm_es import RmES from .gld import GLD from .sim_anneal import SimAnneal +from .snes import SNES +from .xnes import xNES +from .esmc import ESMC +from .des import DES +from .samr_ga import SAMR_GA +from .gesmr_ga import GESMR_GA __all__ = [ @@ -31,7 +36,6 @@ "PGPE", "PBT", "PersistentES", - "xNES", "ARS", "Sep_CMA_ES", "BIPOP_CMA_ES", @@ -43,4 +47,10 @@ "RmES", "GLD", "SimAnneal", + "SNES", + "xNES", + "ESMC", + "DES", + "SAMR_GA", + "GESMR_GA", ] diff --git a/evosax/strategies/cma_es.py b/evosax/strategies/cma_es.py index 59b7ee3..68d28af 100755 --- a/evosax/strategies/cma_es.py +++ b/evosax/strategies/cma_es.py @@ -157,7 +157,9 @@ def ask_strategy( self, rng: chex.PRNGKey, state: EvoState, params: EvoParams ) -> Tuple[chex.Array, EvoState]: """`ask` for new parameter candidates to evaluate next.""" - C, B, D = full_eigen_decomp(state.C, state.B, state.D) + C, B, D = full_eigen_decomp( + state.C, state.B, state.D, state.gen_counter + ) x = sample( rng, state.mean, @@ -197,6 +199,7 @@ def tell_strategy( y_w, params.c_sigma, params.mu_eff, + state.gen_counter, ) p_c, norm_p_sigma, h_sigma = update_p_c( @@ -259,9 +262,10 @@ def update_p_sigma( y_w: chex.Array, c_sigma: float, mu_eff: float, + gen_counter: int, ) -> Tuple[chex.Array, chex.Array, chex.Array, None, None]: """Update evolution path for covariance matrix.""" - C, B, D = full_eigen_decomp(C, B, D) + C, B, D = full_eigen_decomp(C, B, D, gen_counter) C_2 = B.dot(jnp.diag(1 / D)).dot(B.T) # C^(-1/2) = B D^(-1) B^T p_sigma_new = (1 - c_sigma) * p_sigma + jnp.sqrt( c_sigma * (2 - c_sigma) * mu_eff diff --git a/evosax/strategies/esmc.py b/evosax/strategies/esmc.py new file mode 100644 index 0000000..ff573da --- /dev/null +++ b/evosax/strategies/esmc.py @@ -0,0 +1,116 @@ +import jax +import jax.numpy as jnp +import chex +from typing import Tuple +from ..strategy import Strategy +from ..utils import GradientOptimizer, OptState, OptParams +from flax import struct + + +@struct.dataclass +class EvoState: + mean: chex.Array + sigma: chex.Array + opt_state: OptState + best_member: chex.Array + best_fitness: float = jnp.finfo(jnp.float32).max + gen_counter: int = 0 + + +@struct.dataclass +class EvoParams: + opt_params: OptParams + sigma_init: float = 0.03 + sigma_decay: float = 0.999 + sigma_limit: float = 0.01 + sigma_lrate: float = 0.2 # Learning rate for std + sigma_max_change: float = 0.2 # Clip adaptive sigma to 20% + init_min: float = 0.0 + init_max: float = 0.0 + clip_min: float = -jnp.finfo(jnp.float32).max + clip_max: float = jnp.finfo(jnp.float32).max + + +class ESMC(Strategy): + def __init__( + self, + num_dims: int, + popsize: int, + opt_name: str = "adam", + ): + """ESMC (Merchant et al., 2021) + Reference: https://proceedings.mlr.press/v139/merchant21a.html + """ + super().__init__(num_dims, popsize) + assert self.popsize & 1, "Population size must be odd" + assert opt_name in ["sgd", "adam", "rmsprop", "clipup"] + self.optimizer = GradientOptimizer[opt_name](self.num_dims) + self.strategy_name = "ESMC" + + @property + def params_strategy(self) -> EvoParams: + """Return default parameters of evolution strategy.""" + return EvoParams(opt_params=self.optimizer.default_params) + + def initialize_strategy( + self, rng: chex.PRNGKey, params: EvoParams + ) -> EvoState: + """`initialize` the evolution strategy.""" + initialization = jax.random.uniform( + rng, + (self.num_dims,), + minval=params.init_min, + maxval=params.init_max, + ) + state = EvoState( + mean=initialization, + sigma=jnp.ones(self.num_dims) * params.sigma_init, + opt_state=self.optimizer.initialize(params.opt_params), + best_member=initialization, + ) + return state + + def ask_strategy( + self, rng: chex.PRNGKey, state: EvoState, params: EvoParams + ) -> Tuple[chex.Array, EvoState]: + """`ask` for new parameter candidates to evaluate next.""" + # Antithetic sampling of noise + z_plus = jax.random.normal( + rng, + (int(self.popsize / 2), self.num_dims), + ) + z = jnp.concatenate( + [jnp.zeros((1, self.num_dims)), z_plus, -1.0 * z_plus] + ) + x = state.mean + z * state.sigma.reshape(1, self.num_dims) + return x, state + + def tell_strategy( + self, + x: chex.Array, + fitness: chex.Array, + state: EvoState, + params: EvoParams, + ) -> EvoState: + """Update both mean and dim.-wise isotropic Gaussian scale.""" + # Reconstruct noise from last mean/std estimates + noise = (x - state.mean) / state.sigma + bline_fitness = fitness[0] + noise = noise[1:] + fitness = fitness[1:] + noise_1 = noise[: int((self.popsize - 1) / 2)] + fit_1 = fitness[: int((self.popsize - 1) / 2)] + fit_2 = fitness[int((self.popsize - 1) / 2) :] + fit_diff = jnp.minimum(fit_1, bline_fitness) - jnp.minimum( + fit_2, bline_fitness + ) + fit_diff_noise = jnp.dot(noise_1.T, fit_diff) + theta_grad = 1.0 / int((self.popsize - 1) / 2) * fit_diff_noise + # Grad update using optimizer instance - decay lrate if desired + mean, opt_state = self.optimizer.step( + state.mean, theta_grad, state.opt_state, params.opt_params + ) + opt_state = self.optimizer.update(opt_state, params.opt_params) + sigma = state.sigma * params.sigma_decay + sigma = jnp.maximum(sigma, params.sigma_limit) + return state.replace(mean=mean, sigma=sigma, opt_state=opt_state) diff --git a/evosax/strategies/gesmr_ga.py b/evosax/strategies/gesmr_ga.py new file mode 100644 index 0000000..39caad6 --- /dev/null +++ b/evosax/strategies/gesmr_ga.py @@ -0,0 +1,160 @@ +import jax +import jax.numpy as jnp +import chex +from typing import Tuple +from ..strategy import Strategy +from flax import struct + + +@struct.dataclass +class EvoState: + rng: chex.PRNGKey + mean: chex.Array + archive: chex.Array + fitness: chex.Array + sigma: chex.Array + best_member: chex.Array + best_fitness: float = jnp.finfo(jnp.float32).max + gen_counter: int = 0 + + +@struct.dataclass +class EvoParams: + sigma_init: float = 0.07 + sigma_meta: float = 2.0 + init_min: float = 0.0 + init_max: float = 0.0 + clip_min: float = -jnp.finfo(jnp.float32).max + clip_max: float = jnp.finfo(jnp.float32).max + + +class GESMR_GA(Strategy): + def __init__( + self, + num_dims: int, + popsize: int, + elite_ratio: float = 0.5, + sigma_ratio: float = 0.5, + ): + """Self-Adaptation Mutation Rate GA.""" + + super().__init__(num_dims, popsize) + self.elite_ratio = elite_ratio + self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) + self.num_sigma_groups = int(jnp.sqrt(self.popsize)) + self.members_per_group = int( + jnp.ceil(self.popsize / self.num_sigma_groups) + ) + self.sigma_ratio = sigma_ratio + self.sigma_popsize = max( + 1, int(self.num_sigma_groups * self.sigma_ratio) + ) + self.strategy_name = "GESMR_GA" + + @property + def params_strategy(self) -> EvoParams: + """Return default parameters of evolution strategy.""" + return EvoParams() + + def initialize_strategy( + self, rng: chex.PRNGKey, params: EvoParams + ) -> EvoState: + """`initialize` the differential evolution strategy.""" + rng, rng_init = jax.random.split(rng) + initialization = jax.random.uniform( + rng_init, + (self.elite_popsize, self.num_dims), + minval=params.init_min, + maxval=params.init_max, + ) + state = EvoState( + rng=rng, + mean=initialization[0], + archive=initialization, + fitness=jnp.zeros(self.popsize) + jnp.finfo(jnp.float32).max, + sigma=jnp.zeros(self.num_sigma_groups) + params.sigma_init, + best_member=initialization[0], + ) + return state + + def ask_strategy( + self, rng: chex.PRNGKey, state: EvoState, params: EvoParams + ) -> Tuple[chex.Array, EvoState]: + """`ask` for new proposed candidates to evaluate next.""" + rng, rng_idx, rng_eps_x, rng_eps_s = jax.random.split(rng, 4) + # Sample noise for mutation of x and sigma + eps_x = jax.random.normal(rng_eps_x, (self.popsize, self.num_dims)) + eps_s = jax.random.uniform( + rng_eps_s, (self.num_sigma_groups,), minval=-1, maxval=1 + ) + + # Sample members to evaluate from parent archive + idx = jax.random.choice( + rng_idx, jnp.arange(self.elite_popsize), (self.popsize - 1,) + ) + x = jnp.concatenate([state.archive[0][None, :], state.archive[idx]]) + + # Store fitness before perturbation (used to compute meta-fitness) + fitness_mem = jnp.concatenate( + [state.fitness[0][None], state.fitness[idx]] + ) + + # Apply sigma mutation on group level -> repeat for popmember broadcast + sigma_perturb = state.sigma * params.sigma_meta ** eps_s + sigma_repeated = jnp.repeat(sigma_perturb, self.members_per_group)[ + : self.popsize + ] + sigma = jnp.concatenate([state.sigma[0][None], sigma_repeated[1:]]) + + # Apply x mutation -> scale specific to group membership + x += sigma[:, None] * eps_x + return x, state.replace( + archive=x, fitness=fitness_mem, sigma=sigma_perturb + ) + + def tell_strategy( + self, + x: chex.Array, + fitness: chex.Array, + state: EvoState, + params: EvoParams, + ) -> EvoState: + """`tell` update to ES state.""" + # Select best x members + idx = jnp.argsort(fitness)[: self.elite_popsize] + archive = x[idx] + + # Select best sigma based on function value improvement + group_ids = jnp.repeat( + jnp.arange(self.members_per_group), self.num_sigma_groups + )[: self.popsize] + delta_fitness = fitness - state.fitness + + best_deltas = [] + for k in range(self.num_sigma_groups): + sub_mask = group_ids == k + sub_delta = ( + sub_mask * delta_fitness + + (1 - sub_mask) * jnp.finfo(jnp.float32).max + ) + max_sub_delta = jnp.min(sub_delta) + best_deltas.append(max_sub_delta) + + idx_select = jnp.argsort(jnp.array(best_deltas))[: self.sigma_popsize] + sigma_elite = state.sigma[idx_select] + + # Resample sigmas with replacement + rng, rng_sigma = jax.random.split(state.rng) + idx_s = jax.random.choice( + rng_sigma, + jnp.arange(self.sigma_popsize), + (self.num_sigma_groups - 1,), + ) + sigma = jnp.concatenate([state.sigma[0][None], sigma_elite[idx_s]]) + return state.replace( + rng=rng, + fitness=fitness[idx], + archive=archive, + sigma=sigma, + mean=archive[0], + ) diff --git a/evosax/strategies/pgpe.py b/evosax/strategies/pgpe.py index 371da7b..b2741d6 100755 --- a/evosax/strategies/pgpe.py +++ b/evosax/strategies/pgpe.py @@ -36,8 +36,8 @@ def __init__( self, num_dims: int, popsize: int, - elite_ratio: float = 0.1, - opt_name: str = "sgd", + elite_ratio: float = 1.0, + opt_name: str = "adam", ): """PGPE (e.g. Sehnke et al., 2010) Reference: https://tinyurl.com/2p8bn956 @@ -98,9 +98,9 @@ def tell_strategy( """Update both mean and dim.-wise isotropic Gaussian scale.""" # Reconstruct noise from last mean/std estimates noise = (x - state.mean) / state.sigma - noise_1 = noise[: int(self.popsize / 2)] - fit_1 = fitness[: int(self.popsize / 2)] - fit_2 = fitness[int(self.popsize / 2) :] + noise_1 = noise[::2] + fit_1 = fitness[::2] + fit_2 = fitness[1::2] elite_idx = jnp.minimum(fit_1, fit_2).argsort()[: self.elite_popsize] fitness_elite = jnp.concatenate([fit_1[elite_idx], fit_2[elite_idx]]) @@ -120,18 +120,19 @@ def tell_strategy( - (state.sigma * state.sigma).reshape(1, self.num_dims) ) / state.sigma.reshape(1, self.num_dims) rS = (fit_1 + fit_2) / 2.0 - jnp.mean(fitness_elite) - delta_sigma = (jnp.dot(rS, S)) / self.elite_popsize - change_sigma = params.sigma_lrate * delta_sigma - change_sigma = jnp.minimum( - change_sigma, params.sigma_max_change * state.sigma - ) - change_sigma = jnp.maximum( - change_sigma, -params.sigma_max_change * state.sigma - ) + delta_sigma = jnp.dot(rS, S) / (self.elite_popsize / 2) + + allowed_delta = jnp.abs(state.sigma) * params.sigma_max_change + min_allowed = state.sigma - allowed_delta + max_allowed = state.sigma + allowed_delta # adjust sigma according to the adaptive sigma calculation # for stability, don't let sigma move more than 20% of orig value - sigma = state.sigma - change_sigma + sigma = jnp.clip( + state.sigma - params.sigma_lrate * delta_sigma, + min_allowed, + max_allowed, + ) sigma = sigma * params.sigma_decay sigma = jnp.maximum(sigma, params.sigma_limit) return state.replace(mean=mean, sigma=sigma, opt_state=opt_state) diff --git a/evosax/strategies/samr_ga.py b/evosax/strategies/samr_ga.py new file mode 100644 index 0000000..a714005 --- /dev/null +++ b/evosax/strategies/samr_ga.py @@ -0,0 +1,99 @@ +import jax +import jax.numpy as jnp +import chex +from typing import Tuple +from ..strategy import Strategy +from flax import struct + + +@struct.dataclass +class EvoState: + mean: chex.Array + archive: chex.Array + fitness: chex.Array + sigma: chex.Array + best_member: chex.Array + best_fitness: float = jnp.finfo(jnp.float32).max + gen_counter: int = 0 + + +@struct.dataclass +class EvoParams: + sigma_init: float = 0.07 + sigma_meta: float = 2.0 + sigma_best_limit: float = 0.0001 + init_min: float = 0.0 + init_max: float = 0.0 + clip_min: float = -jnp.finfo(jnp.float32).max + clip_max: float = jnp.finfo(jnp.float32).max + + +class SAMR_GA(Strategy): + def __init__(self, num_dims: int, popsize: int, elite_ratio: float = 0.0): + """Self-Adaptation Mutation Rate GA.""" + + super().__init__(num_dims, popsize) + self.elite_ratio = elite_ratio + self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) + self.strategy_name = "SAMR_GA" + + @property + def params_strategy(self) -> EvoParams: + """Return default parameters of evolution strategy.""" + return EvoParams() + + def initialize_strategy( + self, rng: chex.PRNGKey, params: EvoParams + ) -> EvoState: + """`initialize` the differential evolution strategy.""" + initialization = jax.random.uniform( + rng, + (self.elite_popsize, self.num_dims), + minval=params.init_min, + maxval=params.init_max, + ) + state = EvoState( + mean=initialization.mean(axis=0), + archive=initialization, + fitness=jnp.zeros(self.elite_popsize) + jnp.finfo(jnp.float32).max, + sigma=jnp.zeros(self.elite_popsize) + params.sigma_init, + best_member=initialization.mean(axis=0), + ) + return state + + def ask_strategy( + self, rng: chex.PRNGKey, state: EvoState, params: EvoParams + ) -> Tuple[chex.Array, EvoState]: + """`ask` for new proposed candidates to evaluate next.""" + rng, rng_idx, rng_eps_x, rng_eps_s = jax.random.split(rng, 4) + eps_x = jax.random.normal(rng_eps_x, (self.popsize, self.num_dims)) + eps_s = jax.random.uniform( + rng_eps_s, (self.popsize,), minval=-1, maxval=1 + ) + idx = jax.random.choice( + rng_idx, jnp.arange(self.elite_popsize), (self.popsize - 1,) + ) + x = jnp.concatenate([state.archive[0][None, :], state.archive[idx]]) + sigma_0 = jnp.array( + [jnp.maximum(params.sigma_best_limit, state.sigma[0])] + ) + sigma = jnp.concatenate([sigma_0, state.sigma[idx]]) + sigma_gen = sigma * params.sigma_meta ** eps_s + x += sigma_gen[:, None] * eps_x + return x, state.replace(archive=x, sigma=sigma_gen) + + def tell_strategy( + self, + x: chex.Array, + fitness: chex.Array, + state: EvoState, + params: EvoParams, + ) -> EvoState: + """`tell` update to ES state.""" + idx = jnp.argsort(fitness)[: self.elite_popsize] + fitness = fitness[idx] + archive = x[idx] + sigma = state.sigma[idx] + return state.replace( + fitness=fitness, archive=archive, sigma=sigma, mean=archive[0] + ) diff --git a/evosax/strategies/snes.py b/evosax/strategies/snes.py new file mode 100644 index 0000000..b29938d --- /dev/null +++ b/evosax/strategies/snes.py @@ -0,0 +1,101 @@ +import jax +import jax.numpy as jnp +import chex +from typing import Tuple +from ..strategy import Strategy +from flax import struct + + +@struct.dataclass +class EvoState: + mean: chex.Array + sigma: chex.Array + weights: chex.Array + best_member: chex.Array + best_fitness: float = jnp.finfo(jnp.float32).max + gen_counter: int = 0 + + +@struct.dataclass +class EvoParams: + lrate_mean: float = 1.0 + lrate_sigma: float = 1.0 + sigma_init: float = 1.0 + init_min: float = 0.0 + init_max: float = 0.0 + clip_min: float = -jnp.finfo(jnp.float32).max + clip_max: float = jnp.finfo(jnp.float32).max + + +def get_recombination_weights(popsize: int, use_baseline: bool = True): + """Get recombination weights for different ranks.""" + + def get_weight(i): + return jnp.maximum(0, jnp.log(popsize / 2 + 1) - jnp.log(i)) + + weights = jax.vmap(get_weight)(jnp.arange(1, popsize + 1)) + weights_norm = weights / jnp.sum(weights) + return weights_norm - use_baseline * (1 / popsize) + + +class SNES(Strategy): + def __init__(self, num_dims: int, popsize: int): + """Exponential Natural ES (Wierstra et al., 2014) + Reference: https://www.jmlr.org/papers/volume15/wierstra14a/wierstra14a.pdf + """ + super().__init__(num_dims, popsize) + self.strategy_name = "SNES" + + @property + def params_strategy(self) -> EvoParams: + """Return default parameters of evolutionary strategy.""" + lrate_sigma = (3 + jnp.log(self.num_dims)) / ( + 5 * jnp.sqrt(self.num_dims) + ) + params = EvoParams(lrate_sigma=lrate_sigma) + return params + + def initialize_strategy( + self, rng: chex.PRNGKey, params: EvoParams + ) -> EvoState: + """`initialize` the evolutionary strategy.""" + initialization = jax.random.uniform( + rng, + (self.num_dims,), + minval=params.init_min, + maxval=params.init_max, + ) + weights = get_recombination_weights(self.popsize) + state = EvoState( + mean=initialization, + sigma=params.sigma_init * jnp.ones(self.num_dims), + weights=weights.reshape(-1, 1), + best_member=initialization, + ) + + return state + + def ask_strategy( + self, rng: chex.PRNGKey, state: EvoState, params: EvoParams + ) -> Tuple[chex.Array, EvoState]: + """`ask` for new parameter candidates to evaluate next.""" + noise = jax.random.normal(rng, (self.popsize, self.num_dims)) + x = state.mean + noise * state.sigma.reshape(1, self.num_dims) + return x, state + + def tell_strategy( + self, + x: chex.Array, + fitness: chex.Array, + state: EvoState, + params: EvoParams, + ) -> EvoState: + """`tell` performance data for strategy state update.""" + s = (x - state.mean) / state.sigma + ranks = fitness.argsort() + sorted_noise = s[ranks] + grad_mean = (state.weights * sorted_noise).sum(axis=0) + grad_sigma = (state.weights * (sorted_noise ** 2 - 1)).sum(axis=0) + mean = state.mean + params.lrate_mean * state.sigma * grad_mean + sigma = state.sigma * jnp.exp(params.lrate_sigma / 2 * grad_sigma) + return state.replace(mean=mean, sigma=sigma) diff --git a/evosax/strategies/xnes.py b/evosax/strategies/xnes.py index 45744e3..1d6e7d1 100644 --- a/evosax/strategies/xnes.py +++ b/evosax/strategies/xnes.py @@ -4,18 +4,17 @@ from typing import Tuple from ..strategy import Strategy from flax import struct +from .snes import get_recombination_weights @struct.dataclass class EvoState: mean: chex.Array sigma: float - sigma_old: float - amat: chex.Array - bmat: chex.Array + B: chex.Array noise: chex.Array - eta_sigma: float - utilities: chex.Array + lrate_sigma: float + weights: chex.Array best_member: chex.Array best_fitness: float = jnp.finfo(jnp.float32).max gen_counter: int = 0 @@ -23,11 +22,13 @@ class EvoState: @struct.dataclass class EvoParams: - eta_mean: float - eta_sigma_init: float - eta_bmat: float - use_adaptive_sampling: bool = False - use_fitness_shaping: bool = True + lrate_mean: float = 1.0 + lrate_sigma_init: float = 0.1 + lrate_B: float = 0.1 + sigma_init: float = 1.0 + use_adasam: bool = False # Adaptation sampling lrate sigma + rho: float = 0.5 # Significance level adaptation sampling + c_prime: float = 0.1 # Adaptation sampling step size init_min: float = 0.0 init_max: float = 0.0 clip_min: float = -jnp.finfo(jnp.float32).max @@ -45,14 +46,12 @@ def __init__(self, num_dims: int, popsize: int): @property def params_strategy(self) -> EvoParams: """Return default parameters of evolutionary strategy.""" + lrate_sigma = (9 + 3 * jnp.log(self.num_dims)) / ( + 5 * jnp.sqrt(self.num_dims) * self.num_dims + ) + rho = 0.5 - 1.0 / (3 * (self.num_dims + 1)) params = EvoParams( - eta_mean=1.0, - eta_sigma_init=3 - * (3 + jnp.log(self.num_dims)) - * (1.0 / (5 * self.num_dims * jnp.sqrt(self.num_dims))), - eta_bmat=3 - * (3 + jnp.log(self.num_dims)) - * (1.0 / (5 * self.num_dims * jnp.sqrt(self.num_dims))), + lrate_sigma_init=lrate_sigma, lrate_B=lrate_sigma, rho=rho ) return params @@ -60,33 +59,20 @@ def initialize_strategy( self, rng: chex.PRNGKey, params: EvoParams ) -> EvoState: """`initialize` the evolutionary strategy.""" - amat = jnp.eye(self.num_dims) - sigma = abs(jax.scipy.linalg.det(amat)) ** (1.0 / self.num_dims) - bmat = amat * (1.0 / sigma) - # Utility helper for fitness shaping - doesn't work without?! - a = jnp.log(1 + 0.5 * self.popsize) - utilities = jnp.array( - [jnp.maximum(0, a - jnp.log(k)) for k in range(1, self.popsize + 1)] - ) - utilities /= jnp.sum(utilities) - utilities -= 1.0 / self.popsize # broadcast - utilities = utilities[::-1] # ascending order - initialization = jax.random.uniform( rng, (self.num_dims,), minval=params.init_min, maxval=params.init_max, ) + weights = get_recombination_weights(self.popsize) state = EvoState( mean=initialization, - sigma=sigma, - sigma_old=sigma, - amat=amat, - bmat=bmat, + B=jnp.eye(self.num_dims) * params.sigma_init, + sigma=params.sigma_init, noise=jnp.zeros((self.popsize, self.num_dims)), - eta_sigma=params.eta_sigma_init, - utilities=utilities, + lrate_sigma=params.lrate_sigma_init, + weights=weights.reshape(-1, 1), best_member=initialization, ) @@ -97,7 +83,14 @@ def ask_strategy( ) -> Tuple[chex.Array, EvoState]: """`ask` for new parameter candidates to evaluate next.""" noise = jax.random.normal(rng, (self.popsize, self.num_dims)) - x = state.mean + state.sigma * jnp.dot(noise, state.bmat) + + def scale_orient(n, sigma, B): + return state.sigma * state.B.T @ n + + scaled_noise = jax.vmap(scale_orient, in_axes=(0, None, None))( + noise, state.sigma, state.B + ) + x = state.mean + scaled_noise return x, state.replace(noise=noise) def tell_strategy( @@ -108,98 +101,80 @@ def tell_strategy( params: EvoParams, ) -> EvoState: """`tell` performance data for strategy state update.""" - # By default the xNES maximizes the objective - fitness_re = -fitness - isort = fitness_re.argsort() - sorted_fitness = fitness_re[isort] - sorted_noise = state.noise[isort] - sorted_candidates = x[isort] - fitness_shaped = jax.lax.select( - params.use_fitness_shaping, state.utilities, sorted_fitness - ) + ranks = fitness.argsort() + sorted_noise = state.noise[ranks] + grad_mean = (state.weights * sorted_noise).sum(axis=0) - use_adasam = jnp.logical_and( - params.use_adaptive_sampling, state.gen_counter > 1 - ) # sigma_old must be available - eta_sigma = jax.lax.select( - use_adasam, - self.adaptive_sampling( - state.eta_sigma, - state.mean, - state.sigma, - state.bmat, - state.sigma_old, - sorted_candidates, - state.eta_sigma, - ), - state.eta_sigma, - ) + def s_grad_m(weight, noise): + return weight * (noise @ noise.T - jnp.eye(self.num_dims)) - dj_delta = jnp.dot(fitness_shaped, sorted_noise) - dj_mmat = ( - jnp.dot( - sorted_noise.T, - sorted_noise * fitness_shaped.reshape(self.popsize, 1), - ) - - jnp.sum(fitness_shaped) * jnp.eye(self.num_dims) - ) - dj_sigma = jnp.trace(dj_mmat) * (1.0 / self.num_dims) - dj_bmat = dj_mmat - dj_sigma * jnp.eye(self.num_dims) + grad_m = jax.vmap(s_grad_m, in_axes=(0, 0))( + state.weights, sorted_noise + ).sum(axis=0) + grad_sigma = jnp.trace(grad_m) / self.num_dims + grad_B = grad_m - grad_sigma * jnp.eye(self.num_dims) - sigma_old = state.sigma - mean = state.mean + ( - params.eta_mean * state.sigma * jnp.dot(state.bmat, dj_delta) + mean = ( + state.mean + params.lrate_mean * state.sigma * state.B @ grad_mean ) - sigma = sigma_old * jnp.exp(0.5 * eta_sigma * dj_sigma) - bmat = jnp.dot( - state.bmat, - jax.scipy.linalg.expm(0.5 * params.eta_bmat * dj_bmat), + sigma = state.sigma * jnp.exp(state.lrate_sigma / 2 * grad_sigma) + B = state.B * jnp.exp(params.lrate_B / 2 * grad_B) + + lrate_sigma = adaptation_sampling( + state.lrate_sigma, + params.lrate_sigma_init, + mean, + B, + sigma, + state.sigma, + sorted_noise, + params.c_prime, + params.rho, ) - return state.replace( - eta_sigma=eta_sigma, - mean=mean, - sigma=sigma, - bmat=bmat, - sigma_old=sigma_old, + lrate_sigma = jax.lax.select( + params.use_adasam, lrate_sigma, state.lrate_sigma ) - - def adaptive_sampling( - self, - eta_sigma: float, - mu: chex.Array, - sigma: float, - bmat: chex.Array, - sigma_old: float, - z_try: chex.Array, - eta_sigma_init: float, - ) -> float: - """Adaptation sampling.""" - c = 0.1 - rho = 0.5 - 1.0 / (3 * (self.num_dims + 1)) # empirical - - bbmat = jnp.dot(bmat.T, bmat) - cov = sigma ** 2 * bbmat - sigma_ = sigma * jnp.sqrt(sigma * (1.0 / sigma_old)) # increase by 1.5 - cov_ = sigma_ ** 2 * bbmat - - p0 = jax.scipy.stats.multivariate_normal.logpdf(z_try, mean=mu, cov=cov) - p1 = jax.scipy.stats.multivariate_normal.logpdf( - z_try, mean=mu, cov=cov_ + return state.replace( + mean=mean, sigma=sigma, B=B, lrate_sigma=lrate_sigma ) - w = jnp.exp(p1 - p0) - - # Mann-Whitney. It is assumed z_try was in ascending order. - n_ = jnp.sum(w) - u_ = jnp.sum(w * (jnp.arange(self.popsize) + 0.5)) - u_mu = self.popsize * n_ * 0.5 - u_sigma = jnp.sqrt(self.popsize * n_ * (self.popsize + n_ + 1) / 12.0) - cum = jax.scipy.stats.norm.cdf(u_, loc=u_mu, scale=u_sigma) - decrease = cum < rho - eta_out = jax.lax.select( - decrease, - (1 - c) * eta_sigma + c * eta_sigma_init, - jnp.minimum(1, (1 + c) * eta_sigma), - ) - return eta_out +def adaptation_sampling( + lrate_sigma: float, + lrate_sigma_init: float, + mean: chex.Array, + B: chex.Array, + sigma: float, + sigma_old: float, + sorted_noise: chex.Array, + c_prime: float, + rho: float, +) -> float: + """Adaptation sampling on sigma/std learning rate.""" + BB = B.T @ B + A = sigma ** 2 * BB + sigma_prime = sigma * jnp.sqrt(sigma / sigma_old) + A_prime = sigma_prime ** 2 * BB + + # Probability ration and u-test - sorted order assumed for noise + prob_0 = jax.scipy.stats.multivariate_normal.logpdf(sorted_noise, mean, A) + prob_1 = jax.scipy.stats.multivariate_normal.logpdf( + sorted_noise, mean, A_prime + ) + w = jnp.exp(prob_1 - prob_0) + popsize = sorted_noise.shape[0] + n = jnp.sum(w) + u = jnp.sum(w * (jnp.arange(popsize) + 0.5)) + u_mean = popsize * n / 2 + u_sigma = jnp.sqrt(popsize * n * (popsize + n + 1) / 12) + cumulative = jax.scipy.stats.norm.cdf( + u, loc=u_mean + 1e-10, scale=u_sigma + 1e-10 + ) + + # Check test significance and update lrate + lrate_sigma = jax.lax.select( + cumulative < rho, + (1 - c_prime) * lrate_sigma + c_prime * lrate_sigma_init, + jnp.minimum(1, (1 - c_prime) * lrate_sigma), + ) + return lrate_sigma diff --git a/evosax/utils/eigen_decomp.py b/evosax/utils/eigen_decomp.py index 25e42f9..dc26301 100644 --- a/evosax/utils/eigen_decomp.py +++ b/evosax/utils/eigen_decomp.py @@ -4,11 +4,12 @@ def full_eigen_decomp( - C: chex.Array, B: chex.Array, D: chex.Array + C: chex.Array, B: chex.Array, D: chex.Array, gen_counter: int ) -> Tuple[chex.Array, chex.Array, chex.Array]: """Perform eigendecomposition of covariance matrix.""" if B is not None and D is not None: return C, B, D + C = C + 1e-10 * (gen_counter == 0) C = (C + C.T) / 2 # Make sure matrix is symmetric D2, B = jnp.linalg.eigh(C) D = jnp.sqrt(jnp.where(D2 < 0, 1e-20, D2)) diff --git a/evosax/utils/reshape_fitness.py b/evosax/utils/reshape_fitness.py index 4d2205d..dfad93f 100755 --- a/evosax/utils/reshape_fitness.py +++ b/evosax/utils/reshape_fitness.py @@ -9,6 +9,7 @@ def __init__( self, centered_rank: bool = False, z_score: bool = False, + norm_range: bool = False, w_decay: float = 0.0, maximize: bool = False, ): @@ -16,27 +17,30 @@ def __init__( self.w_decay = w_decay self.centered_rank = bool(centered_rank) self.z_score = bool(z_score) + self.norm_range = bool(norm_range) self.maximize = bool(maximize) + # TODO: Add assert statement to check that only one condition is met @partial(jax.jit, static_argnums=(0,)) def apply(self, x: chex.Array, fitness: chex.Array) -> chex.Array: """Max objective trafo, rank shaping, z scoring & add weight decay.""" fitness = jax.lax.select(self.maximize, -1 * fitness, fitness) fitness = jax.lax.select( - self.centered_rank, compute_centered_ranks(fitness), fitness + self.centered_rank, centered_rank_trafo(fitness), fitness ) + fitness = jax.lax.select(self.z_score, z_score_trafo(fitness), fitness) fitness = jax.lax.select( - self.z_score, z_score_fitness(fitness), fitness + self.norm_range, range_norm_trafo(fitness, -1.0, 1.0), fitness ) # "Reduce" fitness based on L2 norm of parameters - l2_fit_red = self.w_decay * compute_weight_norm(x) + l2_fit_red = self.w_decay * compute_l2_norm(x) l2_fit_red = jax.lax.select(self.maximize, -1 * l2_fit_red, l2_fit_red) return fitness + l2_fit_red -def z_score_fitness(fitness: chex.Array) -> chex.Array: +def z_score_trafo(arr: chex.Array) -> chex.Array: """Make fitness 'Gaussian' by substracting mean and dividing by std.""" - return (fitness - jnp.mean(fitness)) / jnp.std(1e-05 + fitness) + return (arr - jnp.mean(arr)) / (jnp.std(arr) + 1e-10) def compute_ranks(fitness: chex.Array) -> chex.Array: @@ -46,13 +50,28 @@ def compute_ranks(fitness: chex.Array) -> chex.Array: return ranks -def compute_centered_ranks(fitness: chex.Array) -> chex.Array: +def centered_rank_trafo(fitness: chex.Array) -> chex.Array: """Return ~ -0.5 to 0.5 centered ranks (best to worst - min!).""" y = compute_ranks(fitness) y /= fitness.size - 1 return y - 0.5 -def compute_weight_norm(x: chex.Array) -> chex.Array: +def compute_l2_norm(x: chex.Array) -> chex.Array: """Compute L2-norm of x_i. Assumes x to have shape (popsize, num_dims).""" return jnp.mean(x * x, axis=1) + + +def range_norm_trafo( + arr: chex.Array, min_val: float = -1.0, max_val: float = 1.0 +) -> chex.Array: + """Map scores into a min/max range.""" + arr = jnp.clip(arr, -1e10, 1e10) + normalized_arr = ( + 2 + * max_val + * (arr - jnp.nanmin(arr)) + / (jnp.nanmax(arr) - jnp.nanmin(arr) + 1e-10) + - min_val + ) + return normalized_arr diff --git a/evosax/utils/reshape_params.py b/evosax/utils/reshape_params.py index aa6fb0c..5ff8160 100755 --- a/evosax/utils/reshape_params.py +++ b/evosax/utils/reshape_params.py @@ -1,17 +1,14 @@ import jax import jax.numpy as jnp import chex -from typing import Union, List, Optional -from flax.core.frozen_dict import FrozenDict, unfreeze -from flax.traverse_util import flatten_dict, unflatten_dict -from jax.tree_util import tree_flatten, tree_unflatten +from typing import Union, Optional +from jax import flatten_util class ParameterReshaper(object): def __init__( self, placeholder_params: Union[chex.ArrayTree, chex.Array], - identity: bool = False, n_devices: Optional[int] = None, verbose: bool = True, ): @@ -19,19 +16,12 @@ def __init__( # Get network shape to reshape self.placeholder_params = placeholder_params - leafs, treedef = jax.tree_util.tree_flatten(placeholder_params) - self._treedef = treedef - self.network_shape = jax.tree_map(jnp.shape, leafs) - self.total_params = get_total_params(self.network_shape) - self.l_id = get_layer_ids(self.network_shape) - - # Special case for no identity mapping (no pytree reshaping) - if identity: - self.reshape = jax.jit(self.reshape_identity) - self.reshape_single = jax.jit(self.reshape_single_flat) - else: - self.reshape = jax.jit(self.reshape_network) - self.reshape_single = jax.jit(self.reshape_single_net) + # Set total parameters depending on type of placeholder params + flat, self.unravel_pytree = flatten_util.ravel_pytree( + placeholder_params + ) + self.total_params = flat.shape[0] + self.reshape_single = jax.jit(self.unravel_pytree) if n_devices is None: self.n_devices = jax.local_device_count() @@ -50,13 +40,9 @@ def __init__( " for optimization." ) - def reshape_identity(self, x: chex.Array) -> chex.Array: - """Return parameters w/o reshaping for evaluation.""" - return x - - def reshape_network(self, x: chex.Array) -> chex.ArrayTree: + def reshape(self, x: chex.Array) -> chex.ArrayTree: """Perform reshaping for a 2D matrix (pop_members, params).""" - vmap_shape = jax.vmap(self.flat_to_network, in_axes=(0,)) + vmap_shape = jax.vmap(self.reshape_single, in_axes=(0,)) if self.n_devices > 1: x = self.split_params_for_pmap(x) map_shape = jax.pmap(vmap_shape) @@ -64,56 +50,12 @@ def reshape_network(self, x: chex.Array) -> chex.ArrayTree: map_shape = vmap_shape return map_shape(x) - def reshape_single_flat(self, x: chex.Array) -> chex.Array: - """Perform reshaping for a 1D vector (params,).""" - return x - - def reshape_single_net(self, x: chex.Array) -> chex.ArrayTree: - """Perform reshaping for a 1D vector (params,).""" - unsqueezed_re = self.flat_to_network(x) - return unsqueezed_re + def split_params_for_pmap(self, param: chex.Array) -> chex.Array: + """Helper reshapes param (bs, #params) into (#dev, bs/#dev, #params).""" + return jnp.stack(jnp.split(param, self.n_devices)) @property def vmap_dict(self) -> chex.ArrayTree: """Get a dictionary specifying axes to vmap over.""" vmap_dict = jax.tree_map(lambda x: 0, self.placeholder_params) return vmap_dict - - def flat_to_network(self, flat_params: chex.Array) -> chex.ArrayTree: - """Fill a FrozenDict with new proposed vector of params.""" - new_nn = list() - - # Loop over layers in network - for i, shape in enumerate(self.network_shape): - # Select params from flat to vector to be reshaped - p_flat = jax.lax.dynamic_slice( - flat_params, (self.l_id[i],), (self.l_id[i + 1] - self.l_id[i],) - ) - # Reshape parameters into matrix/kernel/etc. shape - p_reshaped = p_flat.reshape(shape) - # Place reshaped params into dict and increase counter - new_nn.append(p_reshaped) - return tree_unflatten(self._treedef, new_nn) - - def split_params_for_pmap(self, param: chex.Array) -> chex.Array: - """Helper reshapes param (bs, #params) into (#dev, bs/#dev, #params).""" - return jnp.stack(jnp.split(param, self.n_devices)) - - -def get_total_params(params: List[chex.Array]) -> int: - """Get total number of params in net. Loop over layer modules + params.""" - total_params = 0 - layer_keys = params - # Loop over layers - for l_k in layer_keys: - total_params += jnp.prod(jnp.array(l_k)) - return total_params - - -def get_layer_ids(network_shape_list: List[chex.Array]) -> List[int]: - """Get indices to target when reshaping single flat net into dict.""" - l_id = [0] - for shape in network_shape_list: - add_pcount = jnp.prod(jnp.array(shape)) - l_id.append(int(l_id[-1] + add_pcount)) - return l_id diff --git a/examples/01_classic_benchmark.ipynb b/examples/01_classic_benchmark.ipynb index cd8f39c..6912e2d 100755 --- a/examples/01_classic_benchmark.ipynb +++ b/examples/01_classic_benchmark.ipynb @@ -213,7 +213,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# xNES on Sinusoidal Task" + "# XNES on Sinusoidal Task" ] }, { @@ -239,7 +239,7 @@ } ], "source": [ - "from evosax.strategies import xNES\n", + "from evosax.strategies import XNES\n", "\n", "def f(x):\n", " \"\"\"Taken from https://github.com/chanshing/xnes\"\"\" \n", @@ -249,7 +249,7 @@ "batch_func = jax.vmap(f, in_axes=0)\n", "\n", "rng = jax.random.PRNGKey(0)\n", - "strategy = xNES(popsize=50, num_dims=2)\n", + "strategy = XNES(popsize=50, num_dims=2)\n", "es_params = strategy.default_params\n", "es_params = es_params.replace(use_adaptive_sampling=True, \n", " use_fitness_shaping=True,\n", diff --git a/tests/conftest.py b/tests/conftest.py index a9235a4..6436556 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,6 @@ def pytest_generate_tests(metafunc): "ARS", "PBT", "PersistentES", - "xNES", "Sep_CMA_ES", "Full_iAMaLGaM", "Indep_iAMaLGaM", @@ -26,10 +25,13 @@ def pytest_generate_tests(metafunc): "LM_MA_ES", "RmES", "GLD", + "xNES", + "SNES", + "ESMC", ], ) else: - metafunc.parametrize("strategy_name", ["Full_iAMaLGaM"]) + metafunc.parametrize("strategy_name", ["SNES"]) if "classic_name" in metafunc.fixturenames: if metafunc.config.getoption("all"): diff --git a/tests/test_param_reshape.py b/tests/test_param_reshape.py index 42bb987..be0fbe5 100644 --- a/tests/test_param_reshape.py +++ b/tests/test_param_reshape.py @@ -1,10 +1,21 @@ import jax import jax.numpy as jnp -from flax import linen as nn from evosax.networks import LSTM, MLP, CNN from evosax import ParameterReshaper +def test_flat_vector(): + rng = jax.random.PRNGKey(0) + vec_params = jax.random.normal(rng, (2,)) + reshaper = ParameterReshaper(vec_params) + assert reshaper.total_params == 2 + + # Test population batch matrix reshaping + test_params = jnp.zeros((100, 2)) + out = reshaper.reshape(test_params) + assert out.shape == (100, 2) + + def test_reshape_lstm(): rng = jax.random.PRNGKey(1) network = LSTM( From 6c81bb09987d324008fc7573ec70ed79f07c5527 Mon Sep 17 00:00:00 2001 From: RobertTLange Date: Sat, 19 Nov 2022 14:36:41 +0100 Subject: [PATCH 03/13] Optional param reshaping within strategyies --- evosax/strategies/ars.py | 7 +++--- evosax/strategies/bipop_cma_es.py | 15 ++++++++--- evosax/strategies/cma_es.py | 12 ++++++--- evosax/strategies/de.py | 11 +++++--- evosax/strategies/esmc.py | 7 +++--- evosax/strategies/full_iamalgam.py | 12 ++++++--- evosax/strategies/gesmr_ga.py | 7 +++--- evosax/strategies/gld.py | 11 +++++--- evosax/strategies/indep_iamalgam.py | 12 ++++++--- evosax/strategies/ipop_cma_es.py | 15 ++++++++--- evosax/strategies/lm_ma_es.py | 7 +++--- evosax/strategies/ma_es.py | 12 ++++++--- evosax/strategies/open_es.py | 12 ++++++--- evosax/strategies/pbt.py | 11 +++++--- evosax/strategies/persistent_es.py | 12 ++++++--- evosax/strategies/pgpe.py | 7 +++--- evosax/strategies/pso.py | 11 +++++--- evosax/strategies/rm_es.py | 7 +++--- evosax/strategies/samr_ga.py | 12 ++++++--- evosax/strategies/sep_cma_es.py | 12 ++++++--- evosax/strategies/sim_anneal.py | 11 +++++--- evosax/strategies/simple_es.py | 12 ++++++--- evosax/strategies/simple_ga.py | 12 ++++++--- evosax/strategies/snes.py | 13 +++++++--- evosax/strategies/xnes.py | 11 +++++--- evosax/strategy.py | 39 +++++++++++++++++++++++------ evosax/utils/reshape_fitness.py | 10 +++++--- evosax/utils/reshape_params.py | 31 ++++++++++++++++++++++- 28 files changed, 264 insertions(+), 87 deletions(-) diff --git a/evosax/strategies/ars.py b/evosax/strategies/ars.py index 7e53908..2ef9b7d 100644 --- a/evosax/strategies/ars.py +++ b/evosax/strategies/ars.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from ..utils import GradientOptimizer, OptState, OptParams from flax import struct @@ -32,14 +32,15 @@ class EvoParams: class ARS(Strategy): def __init__( self, - num_dims: int, popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.1, opt_name: str = "sgd", ): """Augmented Random Search (Mania et al., 2018) Reference: https://arxiv.org/pdf/1803.07055.pdf""" - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) assert not self.popsize & 1, "Population size must be even" # ARS performs antithetic sampling & allows you to select # "b" elite perturbation directions for the update diff --git a/evosax/strategies/bipop_cma_es.py b/evosax/strategies/bipop_cma_es.py index 992fed1..6ae84d2 100644 --- a/evosax/strategies/bipop_cma_es.py +++ b/evosax/strategies/bipop_cma_es.py @@ -1,6 +1,6 @@ import jax import chex -from typing import Tuple, Optional +from typing import Tuple, Optional, Union from functools import partial from .cma_es import CMA_ES from ..restarts.restarter import WrapperState, WrapperParams @@ -19,14 +19,23 @@ class RestartParams: class BIPOP_CMA_ES(object): - def __init__(self, num_dims: int, popsize: int, elite_ratio: float = 0.5): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + elite_ratio: float = 0.5, + ): """BIPOP-CMA-ES (Hansen, 2009). Reference: https://hal.inria.fr/inria-00382093/document Inspired by: https://tinyurl.com/44y3ryhf""" self.strategy_name = "BIPOP_CMA_ES" # Instantiate base strategy & wrap it with restart wrapper self.strategy = CMA_ES( - num_dims=num_dims, popsize=popsize, elite_ratio=elite_ratio + num_dims=num_dims, + popsize=popsize, + pholder_params=pholder_params, + elite_ratio=elite_ratio, ) from ..restarts import BIPOP_Restarter from ..restarts.termination import spread_criterion, cma_criterion diff --git a/evosax/strategies/cma_es.py b/evosax/strategies/cma_es.py index 68d28af..4122213 100755 --- a/evosax/strategies/cma_es.py +++ b/evosax/strategies/cma_es.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple, Optional +from typing import Tuple, Optional, Union from ..strategy import Strategy from ..utils.eigen_decomp import full_eigen_decomp from flax import struct @@ -80,11 +80,17 @@ def get_cma_elite_weights( class CMA_ES(Strategy): - def __init__(self, num_dims: int, popsize: int, elite_ratio: float = 0.5): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + elite_ratio: float = 0.5, + ): """CMA-ES (e.g. Hansen, 2016) Reference: https://arxiv.org/abs/1604.00772 Inspired by: https://github.com/CyberAgentAILab/cmaes""" - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) assert 0 <= elite_ratio <= 1 self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) diff --git a/evosax/strategies/de.py b/evosax/strategies/de.py index 338adc2..40f7d07 100755 --- a/evosax/strategies/de.py +++ b/evosax/strategies/de.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from flax import struct @@ -29,11 +29,16 @@ class EvoParams: class DE(Strategy): - def __init__(self, num_dims: int, popsize: int): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + ): """Differential Evolution (Storn & Price, 1997) Reference: https://tinyurl.com/4pje5a74""" assert popsize > 6 - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) self.strategy_name = "DE" @property diff --git a/evosax/strategies/esmc.py b/evosax/strategies/esmc.py index ff573da..ed9f3c5 100644 --- a/evosax/strategies/esmc.py +++ b/evosax/strategies/esmc.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from ..utils import GradientOptimizer, OptState, OptParams from flax import struct @@ -34,14 +34,15 @@ class EvoParams: class ESMC(Strategy): def __init__( self, - num_dims: int, popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, opt_name: str = "adam", ): """ESMC (Merchant et al., 2021) Reference: https://proceedings.mlr.press/v139/merchant21a.html """ - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) assert self.popsize & 1, "Population size must be odd" assert opt_name in ["sgd", "adam", "rmsprop", "clipup"] self.optimizer = GradientOptimizer[opt_name](self.num_dims) diff --git a/evosax/strategies/full_iamalgam.py b/evosax/strategies/full_iamalgam.py index eb6002f..7cdf8f3 100644 --- a/evosax/strategies/full_iamalgam.py +++ b/evosax/strategies/full_iamalgam.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from flax import struct @@ -39,11 +39,17 @@ class EvoParams: class Full_iAMaLGaM(Strategy): - def __init__(self, num_dims: int, popsize: int, elite_ratio: float = 0.35): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + elite_ratio: float = 0.35, + ): """(Iterative) AMaLGaM (Bosman et al., 2013) - Full Covariance Reference: https://tinyurl.com/y9fcccx2 """ - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) assert 0 <= elite_ratio <= 1 self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) diff --git a/evosax/strategies/gesmr_ga.py b/evosax/strategies/gesmr_ga.py index 39caad6..53aaec1 100644 --- a/evosax/strategies/gesmr_ga.py +++ b/evosax/strategies/gesmr_ga.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from flax import struct @@ -31,14 +31,15 @@ class EvoParams: class GESMR_GA(Strategy): def __init__( self, - num_dims: int, popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, sigma_ratio: float = 0.5, ): """Self-Adaptation Mutation Rate GA.""" - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) self.num_sigma_groups = int(jnp.sqrt(self.popsize)) diff --git a/evosax/strategies/gld.py b/evosax/strategies/gld.py index 028bd74..7ca0f53 100644 --- a/evosax/strategies/gld.py +++ b/evosax/strategies/gld.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from flax import struct @@ -26,10 +26,15 @@ class EvoParams: class GLD(Strategy): - def __init__(self, num_dims: int, popsize: int): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + ): """Gradientless Descent (Golovin et al., 2019) Reference: https://arxiv.org/pdf/1911.06317.pdf""" - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) self.strategy_name = "GLD" @property diff --git a/evosax/strategies/indep_iamalgam.py b/evosax/strategies/indep_iamalgam.py index 70faed4..0e0b472 100644 --- a/evosax/strategies/indep_iamalgam.py +++ b/evosax/strategies/indep_iamalgam.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from .full_iamalgam import ( anticipated_mean_shift, @@ -44,11 +44,17 @@ class EvoParams: class Indep_iAMaLGaM(Strategy): - def __init__(self, num_dims: int, popsize: int, elite_ratio: float = 0.35): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + elite_ratio: float = 0.35, + ): """(Iterative) AMaLGaM (Bosman et al., 2013) - Diagonal Covariance Reference: https://tinyurl.com/y9fcccx2 """ - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) assert 0 <= elite_ratio <= 1 self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) diff --git a/evosax/strategies/ipop_cma_es.py b/evosax/strategies/ipop_cma_es.py index bdbb071..e99f6fe 100644 --- a/evosax/strategies/ipop_cma_es.py +++ b/evosax/strategies/ipop_cma_es.py @@ -1,6 +1,6 @@ import jax import chex -from typing import Tuple, Optional +from typing import Tuple, Optional, Union from functools import partial from .cma_es import CMA_ES from ..restarts.restarter import WrapperState, WrapperParams @@ -19,14 +19,23 @@ class RestartParams: class IPOP_CMA_ES(object): - def __init__(self, num_dims: int, popsize: int, elite_ratio: float = 0.5): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + elite_ratio: float = 0.5, + ): """IPOP-CMA-ES (Auer & Hansen, 2005). Reference: http://www.cmap.polytechnique.fr/~nikolaus.hansen/cec2005ipopcmaes.pdf """ self.strategy_name = "IPOP_CMA_ES" # Instantiate base strategy & wrap it with restart wrapper self.strategy = CMA_ES( - num_dims=num_dims, popsize=popsize, elite_ratio=elite_ratio + popsize=popsize, + num_dims=num_dims, + pholder_params=pholder_params, + elite_ratio=elite_ratio, ) from ..restarts import IPOP_Restarter from ..restarts.termination import cma_criterion, spread_criterion diff --git a/evosax/strategies/lm_ma_es.py b/evosax/strategies/lm_ma_es.py index 7c4d033..03586ce 100644 --- a/evosax/strategies/lm_ma_es.py +++ b/evosax/strategies/lm_ma_es.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from .cma_es import get_cma_elite_weights from flax import struct @@ -41,15 +41,16 @@ class EvoParams: class LM_MA_ES(Strategy): def __init__( self, - num_dims: int, popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, memory_size: int = 10, ): """Limited Memory MA-ES (Loshchilov et al., 2017) Reference: https://arxiv.org/pdf/1705.06693.pdf """ - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) assert 0 <= elite_ratio <= 1 self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) diff --git a/evosax/strategies/ma_es.py b/evosax/strategies/ma_es.py index 1e65e92..5674a7f 100644 --- a/evosax/strategies/ma_es.py +++ b/evosax/strategies/ma_es.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from .cma_es import get_cma_elite_weights from flax import struct @@ -36,11 +36,17 @@ class EvoParams: class MA_ES(Strategy): - def __init__(self, num_dims: int, popsize: int, elite_ratio: float = 0.5): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + elite_ratio: float = 0.5, + ): """MA-ES (Bayer & Sendhoff, 2017) Reference: https://www.honda-ri.de/pubs/pdf/3376.pdf """ - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) assert 0 <= elite_ratio <= 1 self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) diff --git a/evosax/strategies/open_es.py b/evosax/strategies/open_es.py index 8461950..ea9a6b5 100755 --- a/evosax/strategies/open_es.py +++ b/evosax/strategies/open_es.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from ..utils import GradientOptimizer, OptState, OptParams from flax import struct @@ -30,11 +30,17 @@ class EvoParams: class OpenES(Strategy): - def __init__(self, num_dims: int, popsize: int, opt_name: str = "adam"): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + opt_name: str = "adam", + ): """OpenAI-ES (Salimans et al. (2017) Reference: https://arxiv.org/pdf/1703.03864.pdf Inspired by: https://github.com/hardmaru/estool/blob/master/es.py""" - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) assert not self.popsize & 1, "Population size must be even" assert opt_name in ["sgd", "adam", "rmsprop", "clipup"] self.optimizer = GradientOptimizer[opt_name](self.num_dims) diff --git a/evosax/strategies/pbt.py b/evosax/strategies/pbt.py index 1f41cdb..035166b 100755 --- a/evosax/strategies/pbt.py +++ b/evosax/strategies/pbt.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from flax import struct @@ -27,10 +27,15 @@ class EvoParams: class PBT(Strategy): - def __init__(self, num_dims: int, popsize: int): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + ): """Synchronous Population-Based Training (Jaderberg et al., 2017) Reference: https://arxiv.org/abs/1711.09846""" - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) self.strategy_name = "PBT" @property diff --git a/evosax/strategies/persistent_es.py b/evosax/strategies/persistent_es.py index 4d9f86f..128bb89 100644 --- a/evosax/strategies/persistent_es.py +++ b/evosax/strategies/persistent_es.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from ..utils import GradientOptimizer, OptState, OptParams from flax import struct @@ -34,12 +34,18 @@ class EvoParams: class PersistentES(Strategy): - def __init__(self, num_dims: int, popsize: int, opt_name: str = "adam"): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + opt_name: str = "adam", + ): """Persistent ES (Vicol et al., 2021). Reference: http://proceedings.mlr.press/v139/vicol21a.html Inspired by: http://proceedings.mlr.press/v139/vicol21a/vicol21a-supp.pdf """ - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) assert not self.popsize & 1, "Population size must be even" assert opt_name in ["sgd", "adam", "rmsprop", "clipup"] self.optimizer = GradientOptimizer[opt_name](self.num_dims) diff --git a/evosax/strategies/pgpe.py b/evosax/strategies/pgpe.py index b2741d6..13d9026 100755 --- a/evosax/strategies/pgpe.py +++ b/evosax/strategies/pgpe.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from ..utils import GradientOptimizer, OptState, OptParams from flax import struct @@ -34,15 +34,16 @@ class EvoParams: class PGPE(Strategy): def __init__( self, - num_dims: int, popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 1.0, opt_name: str = "adam", ): """PGPE (e.g. Sehnke et al., 2010) Reference: https://tinyurl.com/2p8bn956 Inspired by: https://github.com/hardmaru/estool/blob/master/es.py""" - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) assert 0 <= elite_ratio <= 1 self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize / 2 * self.elite_ratio)) diff --git a/evosax/strategies/pso.py b/evosax/strategies/pso.py index 67f11d0..957627d 100755 --- a/evosax/strategies/pso.py +++ b/evosax/strategies/pso.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from flax import struct @@ -31,10 +31,15 @@ class EvoParams: class PSO(Strategy): - def __init__(self, num_dims: int, popsize: int): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + ): """Particle Swarm Optimization (Kennedy & Eberhart, 1995) Reference: https://ieeexplore.ieee.org/document/488968""" - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) self.strategy_name = "PSO" @property diff --git a/evosax/strategies/rm_es.py b/evosax/strategies/rm_es.py index d8bf0b6..9d3755e 100644 --- a/evosax/strategies/rm_es.py +++ b/evosax/strategies/rm_es.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from flax import struct @@ -62,15 +62,16 @@ def get_cma_elite_weights( class RmES(Strategy): def __init__( self, - num_dims: int, popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, memory_size: int = 10, ): """Rank-m ES (Li & Zhang, 2017) Reference: https://ieeexplore.ieee.org/document/8080257 """ - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) assert 0 <= elite_ratio <= 1 self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) diff --git a/evosax/strategies/samr_ga.py b/evosax/strategies/samr_ga.py index a714005..43220be 100644 --- a/evosax/strategies/samr_ga.py +++ b/evosax/strategies/samr_ga.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from flax import struct @@ -29,10 +29,16 @@ class EvoParams: class SAMR_GA(Strategy): - def __init__(self, num_dims: int, popsize: int, elite_ratio: float = 0.0): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + elite_ratio: float = 0.0, + ): """Self-Adaptation Mutation Rate GA.""" - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) self.strategy_name = "SAMR_GA" diff --git a/evosax/strategies/sep_cma_es.py b/evosax/strategies/sep_cma_es.py index 3fbc2c6..95e3917 100644 --- a/evosax/strategies/sep_cma_es.py +++ b/evosax/strategies/sep_cma_es.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple, Optional +from typing import Tuple, Optional, Union from ..strategy import Strategy from ..utils.eigen_decomp import diag_eigen_decomp from flax import struct @@ -57,12 +57,18 @@ def get_cma_elite_weights( class Sep_CMA_ES(Strategy): - def __init__(self, num_dims: int, popsize: int, elite_ratio: float = 0.5): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + elite_ratio: float = 0.5, + ): """Separable CMA-ES (e.g. Ros & Hansen, 2008) Reference: https://hal.inria.fr/inria-00287367/document Inspired by: github.com/CyberAgentAILab/cmaes/blob/main/cmaes/_sepcma.py """ - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) assert 0 <= elite_ratio <= 1 self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) diff --git a/evosax/strategies/sim_anneal.py b/evosax/strategies/sim_anneal.py index 5488abc..5fded53 100644 --- a/evosax/strategies/sim_anneal.py +++ b/evosax/strategies/sim_anneal.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from flax import struct @@ -33,11 +33,16 @@ class EvoParams: class SimAnneal(Strategy): - def __init__(self, num_dims: int, popsize: int): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + ): """Simulated Annealing (Rasdi Rere et al., 2015) Reference: https://www.sciencedirect.com/science/article/pii/S1877050915035759 """ - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) self.strategy_name = "SimAnneal" @property diff --git a/evosax/strategies/simple_es.py b/evosax/strategies/simple_es.py index 077b6b5..31f8d8a 100755 --- a/evosax/strategies/simple_es.py +++ b/evosax/strategies/simple_es.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from flax import struct @@ -28,11 +28,17 @@ class EvoParams: class SimpleES(Strategy): - def __init__(self, num_dims: int, popsize: int, elite_ratio: float = 0.5): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + elite_ratio: float = 0.5, + ): """Simple Gaussian Evolution Strategy (Rechenberg, 1975) Reference: https://onlinelibrary.wiley.com/doi/abs/10.1002/fedr.19750860506 Inspired by: https://github.com/hardmaru/estool/blob/master/es.py""" - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) self.strategy_name = "SimpleES" diff --git a/evosax/strategies/simple_ga.py b/evosax/strategies/simple_ga.py index 0aedc9a..cb4ac98 100755 --- a/evosax/strategies/simple_ga.py +++ b/evosax/strategies/simple_ga.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from flax import struct @@ -30,12 +30,18 @@ class EvoParams: class SimpleGA(Strategy): - def __init__(self, num_dims: int, popsize: int, elite_ratio: float = 0.5): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + elite_ratio: float = 0.5, + ): """Simple Genetic Algorithm (Such et al., 2017) Reference: https://arxiv.org/abs/1712.06567 Inspired by: https://github.com/hardmaru/estool/blob/master/es.py""" - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) self.strategy_name = "SimpleGA" diff --git a/evosax/strategies/snes.py b/evosax/strategies/snes.py index b29938d..70b527b 100644 --- a/evosax/strategies/snes.py +++ b/evosax/strategies/snes.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from flax import struct @@ -39,11 +39,16 @@ def get_weight(i): class SNES(Strategy): - def __init__(self, num_dims: int, popsize: int): - """Exponential Natural ES (Wierstra et al., 2014) + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + ): + """Separable Exponential Natural ES (Wierstra et al., 2014) Reference: https://www.jmlr.org/papers/volume15/wierstra14a/wierstra14a.pdf """ - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) self.strategy_name = "SNES" @property diff --git a/evosax/strategies/xnes.py b/evosax/strategies/xnes.py index 1d6e7d1..7dc6fc5 100644 --- a/evosax/strategies/xnes.py +++ b/evosax/strategies/xnes.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple +from typing import Tuple, Optional, Union from ..strategy import Strategy from flax import struct from .snes import get_recombination_weights @@ -36,11 +36,16 @@ class EvoParams: class xNES(Strategy): - def __init__(self, num_dims: int, popsize: int): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + ): """Exponential Natural ES (Wierstra et al., 2014) Reference: https://www.jmlr.org/papers/volume15/wierstra14a/wierstra14a.pdf Inspired by: https://github.com/chanshing/xnes""" - super().__init__(num_dims, popsize) + super().__init__(popsize, num_dims, pholder_params) self.strategy_name = "xNES" @property diff --git a/evosax/strategy.py b/evosax/strategy.py index c550cd8..9efc899 100755 --- a/evosax/strategy.py +++ b/evosax/strategy.py @@ -1,10 +1,10 @@ import jax import jax.numpy as jnp import chex -from typing import Tuple, Optional +from typing import Tuple, Optional, Union from functools import partial from flax import struct -from .utils import get_best_fitness_member +from .utils import get_best_fitness_member, ParameterReshaper @struct.dataclass @@ -28,11 +28,26 @@ class EvoParams: class Strategy(object): - def __init__(self, num_dims: int, popsize: int): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + ): """Base Class for an Evolution Strategy.""" - self.num_dims = num_dims self.popsize = popsize + # Setup optional parameter reshaper + self.use_param_reshaper = pholder_params is not None + if self.use_param_reshaper: + self.param_reshaper = ParameterReshaper(pholder_params) + self.num_dims = self.param_reshaper.total_params + else: + self.num_dims = num_dims + assert ( + self.num_dims is not None + ), "Provide either num_dims or pholder_params to strategy." + @property def default_params(self) -> EvoParams: """Return default parameters of evolution strategy.""" @@ -58,7 +73,7 @@ def ask( rng: chex.PRNGKey, state: EvoState, params: Optional[EvoParams] = None, - ) -> Tuple[chex.Array, EvoState]: + ) -> Tuple[Union[chex.Array, chex.ArrayTree], EvoState]: """`ask` for new parameter candidates to evaluate next.""" # Use default hyperparameters if no other settings provided if params is None: @@ -68,12 +83,18 @@ def ask( x, state = self.ask_strategy(rng, state, params) # Clip proposal candidates into allowed range x_clipped = jnp.clip(jnp.squeeze(x), params.clip_min, params.clip_max) - return x_clipped, state + + # Reshape parameters into pytrees + if self.use_param_reshaper: + x_out = self.param_reshaper.reshape(x_clipped) + else: + x_out = x_clipped + return x_out, state @partial(jax.jit, static_argnums=(0,)) def tell( self, - x: chex.Array, + x: Union[chex.Array, chex.ArrayTree], fitness: chex.Array, state: EvoState, params: Optional[EvoParams] = None, @@ -83,6 +104,10 @@ def tell( if params is None: params = self.default_params + # Flatten params if using param reshaper for ES update + if self.use_param_reshaper: + x = self.param_reshaper.flatten(x) + # Update the search state based on strategy-specific update state = self.tell_strategy(x, fitness, state, params) diff --git a/evosax/utils/reshape_fitness.py b/evosax/utils/reshape_fitness.py index dfad93f..ab58d82 100755 --- a/evosax/utils/reshape_fitness.py +++ b/evosax/utils/reshape_fitness.py @@ -33,9 +33,13 @@ def apply(self, x: chex.Array, fitness: chex.Array) -> chex.Array: self.norm_range, range_norm_trafo(fitness, -1.0, 1.0), fitness ) # "Reduce" fitness based on L2 norm of parameters - l2_fit_red = self.w_decay * compute_l2_norm(x) - l2_fit_red = jax.lax.select(self.maximize, -1 * l2_fit_red, l2_fit_red) - return fitness + l2_fit_red + if self.w_decay > 0.0: + l2_fit_red = self.w_decay * compute_l2_norm(x) + l2_fit_red = jax.lax.select( + self.maximize, -1 * l2_fit_red, l2_fit_red + ) + fitness += l2_fit_red + return fitness def z_score_trafo(arr: chex.Array) -> chex.Array: diff --git a/evosax/utils/reshape_params.py b/evosax/utils/reshape_params.py index 5ff8160..c3fb5fe 100755 --- a/evosax/utils/reshape_params.py +++ b/evosax/utils/reshape_params.py @@ -2,7 +2,22 @@ import jax.numpy as jnp import chex from typing import Union, Optional -from jax import flatten_util +from jax import vjp, flatten_util +from jax.tree_util import tree_flatten + + +def ravel_pytree(pytree): + leaves, _ = tree_flatten(pytree) + flat, _ = vjp(ravel_list, *leaves) + return flat + + +def ravel_list(*lst): + return ( + jnp.concatenate([jnp.ravel(elt) for elt in lst]) + if lst + else jnp.array([]) + ) class ParameterReshaper(object): @@ -50,6 +65,20 @@ def reshape(self, x: chex.Array) -> chex.ArrayTree: map_shape = vmap_shape return map_shape(x) + def flatten(self, x: chex.ArrayTree) -> chex.Array: + """Reshaping pytree parameters into flat array.""" + vmap_flat = jax.vmap(ravel_pytree) + if self.n_devices > 1: + # Flattening of pmap paramater trees to apply vmap flattening + def map_flat(x): + x_re = jax.tree_map(lambda x: x.reshape(-1, *x.shape[2:]), x) + return vmap_flat(x_re) + + else: + map_flat = vmap_flat + flat = map_flat(x) + return flat + def split_params_for_pmap(self, param: chex.Array) -> chex.Array: """Helper reshapes param (bs, #params) into (#dev, bs/#dev, #params).""" return jnp.stack(jnp.split(param, self.n_devices)) From 2ddcdd57da19f1b5363bb0e816b9dd24a057f36a Mon Sep 17 00:00:00 2001 From: RobertTLange Date: Sat, 19 Nov 2022 15:21:13 +0100 Subject: [PATCH 04/13] Optional fitness shaping within strategy --- evosax/strategies/ars.py | 3 ++- evosax/strategies/bipop_cma_es.py | 2 ++ evosax/strategies/cma_es.py | 3 ++- evosax/strategies/de.py | 3 ++- evosax/strategies/esmc.py | 3 ++- evosax/strategies/full_iamalgam.py | 3 ++- evosax/strategies/gesmr_ga.py | 3 ++- evosax/strategies/gld.py | 3 ++- evosax/strategies/indep_iamalgam.py | 3 ++- evosax/strategies/ipop_cma_es.py | 2 ++ evosax/strategies/lm_ma_es.py | 3 ++- evosax/strategies/ma_es.py | 3 ++- evosax/strategies/open_es.py | 3 ++- evosax/strategies/pbt.py | 3 ++- evosax/strategies/persistent_es.py | 3 ++- evosax/strategies/pgpe.py | 3 ++- evosax/strategies/pso.py | 3 ++- evosax/strategies/rm_es.py | 3 ++- evosax/strategies/samr_ga.py | 3 ++- evosax/strategies/sep_cma_es.py | 3 ++- evosax/strategies/sim_anneal.py | 3 ++- evosax/strategies/simple_es.py | 3 ++- evosax/strategies/simple_ga.py | 3 ++- evosax/strategies/snes.py | 3 ++- evosax/strategies/xnes.py | 3 ++- evosax/strategy.py | 11 +++++++++-- evosax/utils/reshape_fitness.py | 16 +++++++++++----- tests/test_fitness_rollout.py | 3 +-- tests/test_strategy_api.py | 10 ++++++++-- tests/test_strategy_run.py | 10 ++++++++-- 30 files changed, 87 insertions(+), 36 deletions(-) diff --git a/evosax/strategies/ars.py b/evosax/strategies/ars.py index 2ef9b7d..c9b8fd7 100644 --- a/evosax/strategies/ars.py +++ b/evosax/strategies/ars.py @@ -37,10 +37,11 @@ def __init__( pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.1, opt_name: str = "sgd", + **fitness_kwargs: Union[bool, int, float] ): """Augmented Random Search (Mania et al., 2018) Reference: https://arxiv.org/pdf/1803.07055.pdf""" - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) assert not self.popsize & 1, "Population size must be even" # ARS performs antithetic sampling & allows you to select # "b" elite perturbation directions for the update diff --git a/evosax/strategies/bipop_cma_es.py b/evosax/strategies/bipop_cma_es.py index 6ae84d2..067d7a0 100644 --- a/evosax/strategies/bipop_cma_es.py +++ b/evosax/strategies/bipop_cma_es.py @@ -25,6 +25,7 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, + **fitness_kwargs: Union[bool, int, float] ): """BIPOP-CMA-ES (Hansen, 2009). Reference: https://hal.inria.fr/inria-00382093/document @@ -36,6 +37,7 @@ def __init__( popsize=popsize, pholder_params=pholder_params, elite_ratio=elite_ratio, + **fitness_kwargs ) from ..restarts import BIPOP_Restarter from ..restarts.termination import spread_criterion, cma_criterion diff --git a/evosax/strategies/cma_es.py b/evosax/strategies/cma_es.py index 4122213..1b94ee6 100755 --- a/evosax/strategies/cma_es.py +++ b/evosax/strategies/cma_es.py @@ -86,11 +86,12 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, + **fitness_kwargs: Union[bool, int, float] ): """CMA-ES (e.g. Hansen, 2016) Reference: https://arxiv.org/abs/1604.00772 Inspired by: https://github.com/CyberAgentAILab/cmaes""" - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) assert 0 <= elite_ratio <= 1 self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) diff --git a/evosax/strategies/de.py b/evosax/strategies/de.py index 40f7d07..77dc9dd 100755 --- a/evosax/strategies/de.py +++ b/evosax/strategies/de.py @@ -34,11 +34,12 @@ def __init__( popsize: int, num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + **fitness_kwargs: Union[bool, int, float] ): """Differential Evolution (Storn & Price, 1997) Reference: https://tinyurl.com/4pje5a74""" assert popsize > 6 - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) self.strategy_name = "DE" @property diff --git a/evosax/strategies/esmc.py b/evosax/strategies/esmc.py index ed9f3c5..b7dc356 100644 --- a/evosax/strategies/esmc.py +++ b/evosax/strategies/esmc.py @@ -38,11 +38,12 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, opt_name: str = "adam", + **fitness_kwargs: Union[bool, int, float] ): """ESMC (Merchant et al., 2021) Reference: https://proceedings.mlr.press/v139/merchant21a.html """ - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) assert self.popsize & 1, "Population size must be odd" assert opt_name in ["sgd", "adam", "rmsprop", "clipup"] self.optimizer = GradientOptimizer[opt_name](self.num_dims) diff --git a/evosax/strategies/full_iamalgam.py b/evosax/strategies/full_iamalgam.py index 7cdf8f3..23f7643 100644 --- a/evosax/strategies/full_iamalgam.py +++ b/evosax/strategies/full_iamalgam.py @@ -45,11 +45,12 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.35, + **fitness_kwargs: Union[bool, int, float] ): """(Iterative) AMaLGaM (Bosman et al., 2013) - Full Covariance Reference: https://tinyurl.com/y9fcccx2 """ - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) assert 0 <= elite_ratio <= 1 self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) diff --git a/evosax/strategies/gesmr_ga.py b/evosax/strategies/gesmr_ga.py index 53aaec1..acc256d 100644 --- a/evosax/strategies/gesmr_ga.py +++ b/evosax/strategies/gesmr_ga.py @@ -36,10 +36,11 @@ def __init__( pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, sigma_ratio: float = 0.5, + **fitness_kwargs: Union[bool, int, float] ): """Self-Adaptation Mutation Rate GA.""" - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) self.num_sigma_groups = int(jnp.sqrt(self.popsize)) diff --git a/evosax/strategies/gld.py b/evosax/strategies/gld.py index 7ca0f53..2957248 100644 --- a/evosax/strategies/gld.py +++ b/evosax/strategies/gld.py @@ -31,10 +31,11 @@ def __init__( popsize: int, num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + **fitness_kwargs: Union[bool, int, float] ): """Gradientless Descent (Golovin et al., 2019) Reference: https://arxiv.org/pdf/1911.06317.pdf""" - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) self.strategy_name = "GLD" @property diff --git a/evosax/strategies/indep_iamalgam.py b/evosax/strategies/indep_iamalgam.py index 0e0b472..21977b7 100644 --- a/evosax/strategies/indep_iamalgam.py +++ b/evosax/strategies/indep_iamalgam.py @@ -50,11 +50,12 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.35, + **fitness_kwargs: Union[bool, int, float] ): """(Iterative) AMaLGaM (Bosman et al., 2013) - Diagonal Covariance Reference: https://tinyurl.com/y9fcccx2 """ - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) assert 0 <= elite_ratio <= 1 self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) diff --git a/evosax/strategies/ipop_cma_es.py b/evosax/strategies/ipop_cma_es.py index e99f6fe..8795df6 100644 --- a/evosax/strategies/ipop_cma_es.py +++ b/evosax/strategies/ipop_cma_es.py @@ -25,6 +25,7 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, + **fitness_kwargs: Union[bool, int, float] ): """IPOP-CMA-ES (Auer & Hansen, 2005). Reference: http://www.cmap.polytechnique.fr/~nikolaus.hansen/cec2005ipopcmaes.pdf @@ -36,6 +37,7 @@ def __init__( num_dims=num_dims, pholder_params=pholder_params, elite_ratio=elite_ratio, + **fitness_kwargs ) from ..restarts import IPOP_Restarter from ..restarts.termination import cma_criterion, spread_criterion diff --git a/evosax/strategies/lm_ma_es.py b/evosax/strategies/lm_ma_es.py index 03586ce..77d42c2 100644 --- a/evosax/strategies/lm_ma_es.py +++ b/evosax/strategies/lm_ma_es.py @@ -46,11 +46,12 @@ def __init__( pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, memory_size: int = 10, + **fitness_kwargs: Union[bool, int, float] ): """Limited Memory MA-ES (Loshchilov et al., 2017) Reference: https://arxiv.org/pdf/1705.06693.pdf """ - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) assert 0 <= elite_ratio <= 1 self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) diff --git a/evosax/strategies/ma_es.py b/evosax/strategies/ma_es.py index 5674a7f..b4611eb 100644 --- a/evosax/strategies/ma_es.py +++ b/evosax/strategies/ma_es.py @@ -42,11 +42,12 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, + **fitness_kwargs: Union[bool, int, float] ): """MA-ES (Bayer & Sendhoff, 2017) Reference: https://www.honda-ri.de/pubs/pdf/3376.pdf """ - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) assert 0 <= elite_ratio <= 1 self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) diff --git a/evosax/strategies/open_es.py b/evosax/strategies/open_es.py index ea9a6b5..72bba86 100755 --- a/evosax/strategies/open_es.py +++ b/evosax/strategies/open_es.py @@ -36,11 +36,12 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, opt_name: str = "adam", + **fitness_kwargs: Union[bool, int, float] ): """OpenAI-ES (Salimans et al. (2017) Reference: https://arxiv.org/pdf/1703.03864.pdf Inspired by: https://github.com/hardmaru/estool/blob/master/es.py""" - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) assert not self.popsize & 1, "Population size must be even" assert opt_name in ["sgd", "adam", "rmsprop", "clipup"] self.optimizer = GradientOptimizer[opt_name](self.num_dims) diff --git a/evosax/strategies/pbt.py b/evosax/strategies/pbt.py index 035166b..41c7dd9 100755 --- a/evosax/strategies/pbt.py +++ b/evosax/strategies/pbt.py @@ -32,10 +32,11 @@ def __init__( popsize: int, num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + **fitness_kwargs: Union[bool, int, float] ): """Synchronous Population-Based Training (Jaderberg et al., 2017) Reference: https://arxiv.org/abs/1711.09846""" - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) self.strategy_name = "PBT" @property diff --git a/evosax/strategies/persistent_es.py b/evosax/strategies/persistent_es.py index 128bb89..2aa2cde 100644 --- a/evosax/strategies/persistent_es.py +++ b/evosax/strategies/persistent_es.py @@ -40,12 +40,13 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, opt_name: str = "adam", + **fitness_kwargs: Union[bool, int, float] ): """Persistent ES (Vicol et al., 2021). Reference: http://proceedings.mlr.press/v139/vicol21a.html Inspired by: http://proceedings.mlr.press/v139/vicol21a/vicol21a-supp.pdf """ - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) assert not self.popsize & 1, "Population size must be even" assert opt_name in ["sgd", "adam", "rmsprop", "clipup"] self.optimizer = GradientOptimizer[opt_name](self.num_dims) diff --git a/evosax/strategies/pgpe.py b/evosax/strategies/pgpe.py index 13d9026..8da27a7 100755 --- a/evosax/strategies/pgpe.py +++ b/evosax/strategies/pgpe.py @@ -39,11 +39,12 @@ def __init__( pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 1.0, opt_name: str = "adam", + **fitness_kwargs: Union[bool, int, float] ): """PGPE (e.g. Sehnke et al., 2010) Reference: https://tinyurl.com/2p8bn956 Inspired by: https://github.com/hardmaru/estool/blob/master/es.py""" - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) assert 0 <= elite_ratio <= 1 self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize / 2 * self.elite_ratio)) diff --git a/evosax/strategies/pso.py b/evosax/strategies/pso.py index 957627d..35128ce 100755 --- a/evosax/strategies/pso.py +++ b/evosax/strategies/pso.py @@ -36,10 +36,11 @@ def __init__( popsize: int, num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + **fitness_kwargs: Union[bool, int, float] ): """Particle Swarm Optimization (Kennedy & Eberhart, 1995) Reference: https://ieeexplore.ieee.org/document/488968""" - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) self.strategy_name = "PSO" @property diff --git a/evosax/strategies/rm_es.py b/evosax/strategies/rm_es.py index 9d3755e..59f5791 100644 --- a/evosax/strategies/rm_es.py +++ b/evosax/strategies/rm_es.py @@ -67,11 +67,12 @@ def __init__( pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, memory_size: int = 10, + **fitness_kwargs: Union[bool, int, float] ): """Rank-m ES (Li & Zhang, 2017) Reference: https://ieeexplore.ieee.org/document/8080257 """ - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) assert 0 <= elite_ratio <= 1 self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) diff --git a/evosax/strategies/samr_ga.py b/evosax/strategies/samr_ga.py index 43220be..d4a8a49 100644 --- a/evosax/strategies/samr_ga.py +++ b/evosax/strategies/samr_ga.py @@ -35,10 +35,11 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.0, + **fitness_kwargs: Union[bool, int, float] ): """Self-Adaptation Mutation Rate GA.""" - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) self.strategy_name = "SAMR_GA" diff --git a/evosax/strategies/sep_cma_es.py b/evosax/strategies/sep_cma_es.py index 95e3917..d442680 100644 --- a/evosax/strategies/sep_cma_es.py +++ b/evosax/strategies/sep_cma_es.py @@ -63,12 +63,13 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, + **fitness_kwargs: Union[bool, int, float] ): """Separable CMA-ES (e.g. Ros & Hansen, 2008) Reference: https://hal.inria.fr/inria-00287367/document Inspired by: github.com/CyberAgentAILab/cmaes/blob/main/cmaes/_sepcma.py """ - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) assert 0 <= elite_ratio <= 1 self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) diff --git a/evosax/strategies/sim_anneal.py b/evosax/strategies/sim_anneal.py index 5fded53..a805b2b 100644 --- a/evosax/strategies/sim_anneal.py +++ b/evosax/strategies/sim_anneal.py @@ -38,11 +38,12 @@ def __init__( popsize: int, num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + **fitness_kwargs: Union[bool, int, float] ): """Simulated Annealing (Rasdi Rere et al., 2015) Reference: https://www.sciencedirect.com/science/article/pii/S1877050915035759 """ - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) self.strategy_name = "SimAnneal" @property diff --git a/evosax/strategies/simple_es.py b/evosax/strategies/simple_es.py index 31f8d8a..27ed031 100755 --- a/evosax/strategies/simple_es.py +++ b/evosax/strategies/simple_es.py @@ -34,11 +34,12 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, + **fitness_kwargs: Union[bool, int, float] ): """Simple Gaussian Evolution Strategy (Rechenberg, 1975) Reference: https://onlinelibrary.wiley.com/doi/abs/10.1002/fedr.19750860506 Inspired by: https://github.com/hardmaru/estool/blob/master/es.py""" - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) self.strategy_name = "SimpleES" diff --git a/evosax/strategies/simple_ga.py b/evosax/strategies/simple_ga.py index cb4ac98..bfdbbb0 100755 --- a/evosax/strategies/simple_ga.py +++ b/evosax/strategies/simple_ga.py @@ -36,12 +36,13 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, + **fitness_kwargs: Union[bool, int, float] ): """Simple Genetic Algorithm (Such et al., 2017) Reference: https://arxiv.org/abs/1712.06567 Inspired by: https://github.com/hardmaru/estool/blob/master/es.py""" - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) self.strategy_name = "SimpleGA" diff --git a/evosax/strategies/snes.py b/evosax/strategies/snes.py index 70b527b..ce8a82e 100644 --- a/evosax/strategies/snes.py +++ b/evosax/strategies/snes.py @@ -44,11 +44,12 @@ def __init__( popsize: int, num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + **fitness_kwargs: Union[bool, int, float] ): """Separable Exponential Natural ES (Wierstra et al., 2014) Reference: https://www.jmlr.org/papers/volume15/wierstra14a/wierstra14a.pdf """ - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) self.strategy_name = "SNES" @property diff --git a/evosax/strategies/xnes.py b/evosax/strategies/xnes.py index 7dc6fc5..b934524 100644 --- a/evosax/strategies/xnes.py +++ b/evosax/strategies/xnes.py @@ -41,11 +41,12 @@ def __init__( popsize: int, num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + **fitness_kwargs: Union[bool, int, float] ): """Exponential Natural ES (Wierstra et al., 2014) Reference: https://www.jmlr.org/papers/volume15/wierstra14a/wierstra14a.pdf Inspired by: https://github.com/chanshing/xnes""" - super().__init__(popsize, num_dims, pholder_params) + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) self.strategy_name = "xNES" @property diff --git a/evosax/strategy.py b/evosax/strategy.py index 9efc899..834c705 100755 --- a/evosax/strategy.py +++ b/evosax/strategy.py @@ -4,7 +4,7 @@ from typing import Tuple, Optional, Union from functools import partial from flax import struct -from .utils import get_best_fitness_member, ParameterReshaper +from .utils import get_best_fitness_member, ParameterReshaper, FitnessShaper @struct.dataclass @@ -33,6 +33,7 @@ def __init__( popsize: int, num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + **fitness_kwargs: Union[bool, int, float] ): """Base Class for an Evolution Strategy.""" self.popsize = popsize @@ -48,6 +49,9 @@ def __init__( self.num_dims is not None ), "Provide either num_dims or pholder_params to strategy." + # Setup optional fitness shaper + self.fitness_shaper = FitnessShaper(**fitness_kwargs) + @property def default_params(self) -> EvoParams: """Return default parameters of evolution strategy.""" @@ -108,8 +112,11 @@ def tell( if self.use_param_reshaper: x = self.param_reshaper.flatten(x) + # Perform fitness reshaping inside of strategy tell call (if desired) + fitness_re = self.fitness_shaper.apply(x, fitness) + # Update the search state based on strategy-specific update - state = self.tell_strategy(x, fitness, state, params) + state = self.tell_strategy(x, fitness_re, state, params) # Check if there is a new best member & update trackers best_member, best_fitness = get_best_fitness_member(x, fitness, state) diff --git a/evosax/utils/reshape_fitness.py b/evosax/utils/reshape_fitness.py index ab58d82..1f1d7a7 100755 --- a/evosax/utils/reshape_fitness.py +++ b/evosax/utils/reshape_fitness.py @@ -2,16 +2,17 @@ import jax.numpy as jnp import chex from functools import partial +from typing import Union class FitnessShaper(object): def __init__( self, - centered_rank: bool = False, - z_score: bool = False, - norm_range: bool = False, + centered_rank: Union[bool, int] = False, + z_score: Union[bool, int] = False, + norm_range: Union[bool, int] = False, w_decay: float = 0.0, - maximize: bool = False, + maximize: Union[bool, int] = False, ): """JAX-compatible fitness shaping tool.""" self.w_decay = w_decay @@ -19,7 +20,12 @@ def __init__( self.z_score = bool(z_score) self.norm_range = bool(norm_range) self.maximize = bool(maximize) - # TODO: Add assert statement to check that only one condition is met + + # Check that only single fitness shaping transformation is used + num_options_on = self.centered_rank + self.z_score + self.norm_range + assert ( + num_options_on < 2 + ), "Only use one fitness shaping transformation." @partial(jax.jit, static_argnums=(0,)) def apply(self, x: chex.Array, fitness: chex.Array) -> chex.Array: diff --git a/tests/test_fitness_rollout.py b/tests/test_fitness_rollout.py index cce595d..2f35564 100644 --- a/tests/test_fitness_rollout.py +++ b/tests/test_fitness_rollout.py @@ -168,8 +168,7 @@ def test_sequence_fitness(): network.initialize_carry, ) - strategy = ARS(param_reshaper.total_params, 4) - (param_reshaper.total_params) + strategy = ARS(4, param_reshaper.total_params) es_state = strategy.initialize(rng) x, es_state = strategy.ask(rng, es_state) diff --git a/tests/test_strategy_api.py b/tests/test_strategy_api.py index a00f5b4..81812af 100644 --- a/tests/test_strategy_api.py +++ b/tests/test_strategy_api.py @@ -6,7 +6,10 @@ def test_strategy_ask(strategy_name): # Loop over all strategies and test ask API rng = jax.random.PRNGKey(0) - popsize = 20 + if strategy_name == "ESMC": + popsize = 21 + else: + popsize = 20 strategy = Strategies[strategy_name](popsize=popsize, num_dims=2) params = strategy.default_params state = strategy.initialize(rng, params) @@ -19,7 +22,10 @@ def test_strategy_ask(strategy_name): def test_strategy_ask_tell(strategy_name): # Loop over all strategies and test ask API rng = jax.random.PRNGKey(0) - popsize = 20 + if strategy_name == "ESMC": + popsize = 21 + else: + popsize = 20 strategy = Strategies[strategy_name](popsize=popsize, num_dims=2) params = strategy.default_params state = strategy.initialize(rng, params) diff --git a/tests/test_strategy_run.py b/tests/test_strategy_run.py index 59b117c..07eedf2 100644 --- a/tests/test_strategy_run.py +++ b/tests/test_strategy_run.py @@ -13,7 +13,10 @@ def test_strategy_run(strategy_name): rng = jax.random.PRNGKey(0) Strat = Strategies[strategy_name] # PBT also returns copy ID integer - treat separately - popsize = 20 + if strategy_name == "ESMC": + popsize = 21 + else: + popsize = 20 evaluator = ClassicFitness("rosenbrock", 2) fitness_shaper = FitnessShaper() @@ -39,7 +42,10 @@ def test_strategy_scan(strategy_name): rng = jax.random.PRNGKey(0) Strat = Strategies[strategy_name] # PBT also returns copy ID integer - treat separately - popsize = 20 + if strategy_name == "ESMC": + popsize = 21 + else: + popsize = 20 evaluator = ClassicFitness("rosenbrock", 2) fitness_shaper = FitnessShaper() From eeac493c139baade7a4b65134a4cbf57df3259cf Mon Sep 17 00:00:00 2001 From: RobertTLange Date: Sun, 20 Nov 2022 16:57:37 +0100 Subject: [PATCH 05/13] Add Visualizer for BBOB & remove Brax --- evosax/problems/__init__.py | 9 +- evosax/problems/classic.py | 140 ----------- evosax/problems/control_brax.py | 233 ------------------ evosax/problems/modified_ant.py | 423 -------------------------------- evosax/problems/obs_norm.py | 144 ----------- evosax/utils/evojax_wrapper.py | 53 ++++ evosax/utils/visualizer_2d.py | 207 ++++++++++++++++ 7 files changed, 263 insertions(+), 946 deletions(-) delete mode 100755 evosax/problems/classic.py delete mode 100644 evosax/problems/control_brax.py delete mode 100644 evosax/problems/modified_ant.py delete mode 100644 evosax/problems/obs_norm.py create mode 100644 evosax/utils/evojax_wrapper.py create mode 100644 evosax/utils/visualizer_2d.py diff --git a/evosax/problems/__init__.py b/evosax/problems/__init__.py index 8b3fdb6..1726592 100755 --- a/evosax/problems/__init__.py +++ b/evosax/problems/__init__.py @@ -1,22 +1,19 @@ -from .control_brax import BraxFitness from .control_gym import GymFitness from .vision import VisionFitness -from .classic import ClassicFitness +from .bbob import BBOBFitness from .sequence import SequenceFitness ProblemMapper = { "Gym": GymFitness, - "Brax": BraxFitness, "Vision": VisionFitness, - "Classic": ClassicFitness, + "BBOB": BBOBFitness, "Sequence": SequenceFitness, } __all__ = [ - "BraxFitness", "GymFitness", "VisionFitness", - "ClassicFitness", + "BBOBFitness", "SequenceFitness", "ProblemMapper", ] diff --git a/evosax/problems/classic.py b/evosax/problems/classic.py deleted file mode 100755 index a85d294..0000000 --- a/evosax/problems/classic.py +++ /dev/null @@ -1,140 +0,0 @@ -import jax -import jax.numpy as jnp -import chex -from functools import partial - - -class ClassicFitness(object): - def __init__( - self, - fct_name: str = "rosenbrock", - num_dims: int = 2, - num_rollouts: int = 1, - noise_std: float = 0.0, - ): - self.fct_name = fct_name - self.num_dims = num_dims - self.num_rollouts = num_rollouts - # Optional - add Gaussian noise to evaluation fitness - self.noise_std = noise_std - assert self.num_dims >= 2 - - # Use default settings for classic BBOB evaluation functions - if self.fct_name == "quadratic": - self.eval = jax.vmap(quadratic_d_dim, 0) - elif self.fct_name == "rosenbrock": - fn = partial(rosenbrock_d_dim, params={"a": 1, "b": 100}) - self.eval = jax.vmap(fn, 0) - elif self.fct_name == "ackley": - fn = partial( - ackley_d_dim, params={"c": 20, "d": 0.2, "e": 2 * jnp.pi} - ) - self.eval = jax.vmap(fn, 0) - elif self.fct_name == "griewank": - self.eval = jax.vmap(griewank_d_dim, 0) - elif self.fct_name == "rastrigin": - fn = partial(rastrigin_d_dim, params={"f": 10}) - self.eval = jax.vmap(fn, 0) - elif self.fct_name == "schwefel": - self.eval = jax.vmap(schwefel_d_dim, 0) - elif self.fct_name == "himmelblau": - assert self.num_dims == 2 - self.eval = jax.vmap(himmelblau_2_dim, 0) - elif self.fct_name == "six-hump": - assert self.num_dims == 2 - self.eval = jax.vmap(six_hump_camel_2_dim, 0) - else: - raise ValueError("Please provide a valid problem name.") - - @partial(jax.jit, static_argnums=(0,)) - def rollout( - self, rng_input: chex.PRNGKey, eval_params: chex.Array - ) -> chex.Array: - """Batch evaluate the proposal points.""" - fitness = self.eval(eval_params).reshape(eval_params.shape[0], 1) - noise = self.noise_std * jax.random.normal( - rng_input, (eval_params.shape[0], self.num_rollouts) - ) - return (fitness + noise).squeeze() - - -def himmelblau_2_dim(x: chex.Array) -> chex.Array: - """ - 2-dim. Himmelblau function. - f(x*)=0 - Minima at [3, 2], [-2.81, 3.13], - [-3.78, -3.28], [3.58, -1.85] - """ - return (x[0] ** 2 + x[1] - 11) ** 2 + (x[0] + x[1] ** 2 - 7) ** 2 - - -def six_hump_camel_2_dim(x: chex.Array) -> chex.Array: - """ - 2-dim. 6-Hump Camel function. - f(x*)=-1.0316 - Minimum at [0.0898, -0.7126], [-0.0898, 0.7126] - """ - p1 = (4 - 2.1 * x[0] ** 2 + x[0] ** 4 / 3) * x[0] ** 2 - p2 = x[0] * x[1] - p3 = (-4 + 4 * x[1] ** 2) * x[1] ** 2 - return p1 + p2 + p3 - - -def quadratic_d_dim(x: chex.Array) -> chex.Array: - """ - Simple D-dim. quadratic function. - f(x*)=0 - Minimum at [0.]ˆd - """ - return jnp.sum(jnp.square(x)) - - -def rosenbrock_d_dim(x: chex.Array, params: dict) -> chex.Array: - """ - D-Dim. Rosenbrock function. x_i ∈ [-32.768, 32.768] or x_i ∈ [-5, 10] - f(x*)=0 - Minumum at x*=a - """ - x_i, x_sq, x_p = x[:-1], x[:-1] ** 2, x[1:] - return jnp.sum((params["a"] - x_i) ** 2 + params["b"] * (x_p - x_sq) ** 2) - - -def ackley_d_dim(x: chex.Array, params: dict) -> chex.Array: - """ - D-Dim. Ackley function. x_i ∈ [-32.768, 32.768] - f(x*)=0 - Minimum at x*=[0,...,0] - """ - return ( - -params["c"] * jnp.exp(-params["d"] * jnp.sqrt(jnp.mean(x ** 2))) - - jnp.exp(jnp.mean(jnp.cos(params["e"] * x))) - + params["c"] - + jnp.exp(1) - ) - - -def griewank_d_dim(x: chex.Array) -> chex.Array: - """ - D-Dim. Griewank function. x_i ∈ [-600, 600] - f(x*)=0 - Minimum at x*=[0,...,0] - """ - return ( - jnp.sum(x ** 2 / 4000) - - jnp.prod(jnp.cos(x / jnp.sqrt(jnp.arange(1, x.shape[0] + 1)))) - + 1 - ) - - -def rastrigin_d_dim(x: chex.Array, params: dict) -> chex.Array: - """ - D-Dim. Rastrigin function. x_i ∈ [-5.12, 5.12] - f(x*)=0 - Minimum at x*=[0,...,0] - """ - return params["f"] * x.shape[0] + jnp.sum( - x ** 2 - params["f"] * jnp.cos(2 * jnp.pi * x) - ) - - -def schwefel_d_dim(x: chex.Array) -> chex.Array: - """ - D-Dim. Schwefel function. x_i ∈ [-500, 500] - f(x*)=0 - Minimum at x*=[420.9687,...,420.9687] - """ - return 418.9829 * x.shape[0] - jnp.sum( - x * jnp.sin(jnp.sqrt(jnp.absolute(x))) - ) diff --git a/evosax/problems/control_brax.py b/evosax/problems/control_brax.py deleted file mode 100644 index e99fa96..0000000 --- a/evosax/problems/control_brax.py +++ /dev/null @@ -1,233 +0,0 @@ -import jax -import jax.numpy as jnp -import chex -from typing import Optional -from .obs_norm import ObsNormalizer - - -class BraxFitness(object): - def __init__( - self, - env_name: str = "ant", - num_env_steps: int = 1000, - num_rollouts: int = 16, - legacy_spring: bool = True, - normalize: bool = False, - modify_dict: dict = {"torso_mass": 15}, - test: bool = False, - n_devices: Optional[int] = None, - ): - try: - from brax import envs - except ImportError: - raise ImportError( - "You need to install `brax` to use its fitness rollouts." - ) - self.env_name = env_name - self.num_env_steps = num_env_steps - self.num_rollouts = num_rollouts - self.steps_per_member = num_env_steps * num_rollouts - self.test = test - - if self.env_name in [ - "ant", - "halfcheetah", - "hopper", - "humanoid", - "reacher", - "walker2d", - "fetch", - "grasp", - "ur5e", - ]: - # Define the RL environment & network forward fucntion - self.env = envs.create( - env_name=self.env_name, - episode_length=num_env_steps, - legacy_spring=legacy_spring, - ) - elif self.env_name == "modified-ant": - from .modified_ant import create_modified_ant_env - - self.env = create_modified_ant_env(modify_dict) - - self.action_shape = self.env.action_size - self.input_shape = (self.env.observation_size,) - self.obs_normalizer = ObsNormalizer( - self.input_shape, dummy=not normalize - ) - self.obs_params = self.obs_normalizer.get_init_params() - if n_devices is None: - self.n_devices = jax.local_device_count() - else: - self.n_devices = n_devices - - # Keep track of total steps executed in environment - self.total_env_steps = 0 - - def set_apply_fn(self, map_dict, network_apply, carry_init=None): - """Set the network forward function.""" - self.network = network_apply - # Set rollout function based on model architecture - if carry_init is not None: - self.single_rollout = self.rollout_rnn - self.carry_init = carry_init - else: - self.single_rollout = self.rollout_ffw - - # vmap over stochastic evaluations - self.rollout_repeats = jax.vmap(self.single_rollout, in_axes=(0, None)) - self.rollout_pop = jax.vmap( - self.rollout_repeats, in_axes=(None, map_dict) - ) - # pmap over popmembers if > 1 device is available - otherwise pmap - if self.n_devices > 1: - self.rollout_map = self.rollout_pmap - print( - f"BraxFitness: {self.n_devices} devices detected. Please make" - " sure that the ES population size divides evenly across the" - " number of devices to pmap/parallelize over." - ) - else: - self.rollout_map = self.rollout_pop - - def rollout_pmap(self, rng_input, policy_params): - """Parallelize rollout across devices. Split keys/reshape correctly.""" - keys_pmap = jnp.tile(rng_input, (self.n_devices, 1, 1)) - rew_dev, obs_dev, masks_dev = jax.pmap(self.rollout_pop)( - keys_pmap, policy_params - ) - rew_re = rew_dev.reshape(-1, self.num_rollouts) - obs_re = obs_dev.reshape( - -1, self.num_rollouts, self.num_env_steps, self.env.observation_size - ) - masks_re = masks_dev.reshape( - -1, self.num_rollouts, self.num_env_steps, 1 - ) - return rew_re, obs_re, masks_re - - def rollout(self, rng_input, policy_params): - """Placeholder fn call for rolling out a population for multi-evals.""" - rng_pop = jax.random.split(rng_input, self.num_rollouts) - scores, all_obs, masks = jax.jit(self.rollout_map)( - rng_pop, policy_params - ) - # Update normalization parameters if train case! - if not self.test: - obs_re = all_obs.reshape( - self.num_env_steps, -1, self.input_shape[0] - ) - masks_re = masks.reshape(self.num_env_steps, -1) - self.obs_params = self.obs_normalizer.update_normalization_params( - obs_buffer=obs_re, - obs_mask=masks_re, - obs_params=self.obs_params, - ) - - # obs_steps = self.obs_params[0] - # running_mean, running_var = jnp.split(self.obs_params[1:], 2) - # print( - # float(scores.mean()), - # float(masks.mean()), - # obs_steps, - # running_mean.mean(), - # running_var.mean() / (obs_steps + 1), - # ) - - # Update total step counter using only transitions before termination - self.total_env_steps += masks_re.sum() - return scores - - def rollout_ffw( - self, rng_input: chex.PRNGKey, policy_params: chex.ArrayTree - ) -> chex.Array: - """Rollout a jitted brax episode with lax.scan for a feedforward policy.""" - # Reset the environment - rng, rng_reset = jax.random.split(rng_input) - state = self.env.reset(rng_reset) - - def policy_step(state_input, tmp): - """lax.scan compatible step transition in jax env.""" - state, policy_params, rng, cum_reward, valid_mask = state_input - rng, rng_net = jax.random.split(rng) - org_obs = state.obs - norm_obs = self.obs_normalizer.normalize_obs( - org_obs, self.obs_params - ) - action = self.network(policy_params, norm_obs, rng=rng_net) - next_s = self.env.step(state, action) - new_cum_reward = cum_reward + next_s.reward * valid_mask - new_valid_mask = valid_mask * (1 - next_s.done.ravel()) - carry = [next_s, policy_params, rng, new_cum_reward, new_valid_mask] - return carry, [new_valid_mask, org_obs] - - # Scan over episode step loop - carry_out, scan_out = jax.lax.scan( - policy_step, - [state, policy_params, rng, jnp.array([0.0]), jnp.array([1.0])], - (), - self.num_env_steps, - ) - # Return masked sum of rewards accumulated by agent in episode - ep_mask, all_obs = scan_out[0], scan_out[1] - cum_return = carry_out[-2].squeeze() - return cum_return, all_obs, ep_mask - - def rollout_rnn( - self, rng_input: chex.PRNGKey, policy_params: chex.ArrayTree - ) -> chex.Array: - """Rollout a jitted episode with lax.scan for a recurrent policy.""" - # Reset the environment - rng, rng_reset = jax.random.split(rng_input) - state = self.env.reset(rng_reset) - hidden = self.carry_init() - - def policy_step(state_input, tmp): - """lax.scan compatible step transition in jax env.""" - ( - state, - policy_params, - rng, - hidden, - cum_reward, - valid_mask, - ) = state_input - rng, rng_net = jax.random.split(rng) - org_obs = state.obs - norm_obs = self.obs_normalizer.normalize_obs( - state.obs, self.obs_params - ) - hidden, action = self.network( - policy_params, norm_obs, hidden, rng_net - ) - next_s = self.env.step(state, action) - new_cum_reward = cum_reward + next_s.reward * valid_mask - new_valid_mask = valid_mask * (1 - next_s.done.ravel()) - carry = [ - next_s, - policy_params, - rng, - hidden, - new_cum_reward, - new_valid_mask, - ] - return carry, [new_valid_mask, org_obs] - - # Scan over episode step loop - carry_out, scan_out = jax.lax.scan( - policy_step, - [ - state, - policy_params, - rng, - hidden, - jnp.array([0.0]), - jnp.array([1.0]), - ], - (), - self.num_env_steps, - ) - # Return masked sum of rewards accumulated by agent in episode - ep_mask, all_obs = scan_out[0], scan_out[1] - cum_return = carry_out[-2].squeeze() - return cum_return, all_obs, ep_mask diff --git a/evosax/problems/modified_ant.py b/evosax/problems/modified_ant.py deleted file mode 100644 index 8fabe72..0000000 --- a/evosax/problems/modified_ant.py +++ /dev/null @@ -1,423 +0,0 @@ -"""Trains an ant to run in the +x direction. -Adapted from https://raw.githubusercontent.com/google/brax/main/brax/envs/ant.py -Increases mass of main torso body from 10 to 15. -""" - -import brax -from brax import jumpy as jp -from brax.envs import env -from brax.envs import wrappers -from typing import Optional - - -class ModifiedAnt(env.Env): - """Trains an ant to run in the +x direction.""" - - def __init__(self, config, **kwargs): - super().__init__(config=config, **kwargs) - - def reset(self, rng: jp.ndarray) -> env.State: - """Resets the environment to an initial state.""" - rng, rng1, rng2 = jp.random_split(rng, 3) - qpos = self.sys.default_angle() + jp.random_uniform( - rng1, (self.sys.num_joint_dof,), -0.1, 0.1 - ) - qvel = jp.random_uniform(rng2, (self.sys.num_joint_dof,), -0.1, 0.1) - qp = self.sys.default_qp(joint_angle=qpos, joint_velocity=qvel) - info = self.sys.info(qp) - obs = self._get_obs(qp, info) - reward, done, zero = jp.zeros(3) - metrics = { - "reward_ctrl_cost": zero, - "reward_contact_cost": zero, - "reward_forward": zero, - "reward_survive": zero, - } - return env.State(qp, obs, reward, done, metrics) - - def step(self, state: env.State, action: jp.ndarray) -> env.State: - """Run one timestep of the environment's dynamics.""" - qp, info = self.sys.step(state.qp, action) - obs = self._get_obs(qp, info) - - x_before = state.qp.pos[0, 0] - x_after = qp.pos[0, 0] - forward_reward = (x_after - x_before) / self.sys.config.dt - ctrl_cost = 0.5 * jp.sum(jp.square(action)) - contact_cost = ( - 0.5 * 1e-3 * jp.sum(jp.square(jp.clip(info.contact.vel, -1, 1))) - ) - survive_reward = jp.float32(1) - reward = forward_reward - ctrl_cost - contact_cost + survive_reward - - done = jp.where(qp.pos[0, 2] < 0.2, x=jp.float32(1), y=jp.float32(0)) - done = jp.where(qp.pos[0, 2] > 1.0, x=jp.float32(1), y=done) - state.metrics.update( - reward_ctrl_cost=ctrl_cost, - reward_contact_cost=contact_cost, - reward_forward=forward_reward, - reward_survive=survive_reward, - ) - - return state.replace(qp=qp, obs=obs, reward=reward, done=done) - - def _get_obs(self, qp: brax.QP, info: brax.Info) -> jp.ndarray: - """Observe ant body position and velocities.""" - # some pre-processing to pull joint angles and velocities - (joint_angle,), (joint_vel,) = self.sys.joints[0].angle_vel(qp) - - # qpos: - # Z of the torso (1,) - # orientation of the torso as quaternion (4,) - # joint angles (8,) - qpos = [qp.pos[0, 2:], qp.rot[0], joint_angle] - - # qvel: - # velocity of the torso (3,) - # angular velocity of the torso (3,) - # joint angle velocities (8,) - qvel = [qp.vel[0], qp.ang[0], joint_vel] - - # external contact forces: - # delta velocity (3,), delta ang (3,) * 10 bodies in the system - # Note that mujoco has 4 extra bodies tucked inside the Torso that Brax - # ignores - cfrc = [ - jp.clip(info.contact.vel, -1, 1), - jp.clip(info.contact.ang, -1, 1), - ] - # flatten bottom dimension - cfrc = [jp.reshape(x, x.shape[:-2] + (-1,)) for x in cfrc] - - return jp.concatenate(qpos + qvel + cfrc) - - -_CONFIG_MODIFIED = """ -bodies {{ - name: "$ Torso" - colliders {{ - capsule {{ - radius: 0.25 - length: 0.5 - end: 1 - }} - }} - inertia {{ x: 1.0 y: 1.0 z: 1.0 }} - mass: {torso_mass} -}} -bodies {{ - name: "Aux 1" - colliders {{ - rotation {{ x: 90 y: -45 }} - capsule {{ - radius: 0.08 - length: 0.4428427219390869 - }} - }} - inertia {{ x: 1.0 y: 1.0 z: 1.0 }} - mass: 1 -}} -bodies {{ - name: "$ Body 4" - colliders {{ - rotation {{ x: 90 y: -45 }} - capsule {{ - radius: 0.08 - length: 0.7256854176521301 - end: -1 - }} - }} - inertia {{ x: 1.0 y: 1.0 z: 1.0 }} - mass: 1 -}} -bodies {{ - name: "Aux 2" - colliders {{ - rotation {{ x: 90 y: 45 }} - capsule {{ - radius: 0.08 - length: 0.4428427219390869 - }} - }} - inertia {{ x: 1.0 y: 1.0 z: 1.0 }} - mass: 1 -}} -bodies {{ - name: "$ Body 7" - colliders {{ - rotation {{ x: 90 y: 45 }} - capsule {{ - radius: 0.08 - length: 0.7256854176521301 - end: -1 - }} - }} - inertia {{ x: 1.0 y: 1.0 z: 1.0 }} - mass: 1 -}} -bodies {{ - name: "Aux 3" - colliders {{ - rotation {{ x: -90 y: 45 }} - capsule {{ - radius: 0.08 - length: 0.4428427219390869 - }} - }} - inertia {{ x: 1.0 y: 1.0 z: 1.0 }} - mass: 1 -}} -bodies {{ - name: "$ Body 10" - colliders {{ - rotation {{ x: -90 y: 45 }} - capsule {{ - radius: 0.08 - length: 0.7256854176521301 - end: -1 - }} - }} - inertia {{ x: 1.0 y: 1.0 z: 1.0 }} - mass: 1 -}} -bodies {{ - name: "Aux 4" - colliders {{ - rotation {{ x: -90 y: -45 }} - capsule {{ - radius: 0.08 - length: 0.4428427219390869 - }} - }} - inertia {{ x: 1.0 y: 1.0 z: 1.0 }} - mass: 1 -}} -bodies {{ - name: "$ Body 13" - colliders {{ - rotation {{ x: -90 y: -45 }} - capsule {{ - radius: 0.08 - length: 0.7256854176521301 - end: -1 - }} - }} - inertia {{ x: 1.0 y: 1.0 z: 1.0 }} - mass: 1 -}} -bodies {{ - name: "Ground" - colliders {{ - plane {{}} - }} - inertia {{ x: 1.0 y: 1.0 z: 1.0 }} - mass: 1 - frozen {{ all: true }} -}} -joints {{ - name: "$ Torso_Aux 1" - parent_offset {{ x: 0.2 y: 0.2 }} - child_offset {{ x: -0.1 y: -0.1 }} - parent: "$ Torso" - child: "Aux 1" - stiffness: 18000.0 - angular_damping: 20 - spring_damping: 80 - angle_limit {{ min: -30.0 max: 30.0 }} - rotation {{ y: -90 }} -}} -joints {{ - name: "Aux 1_$ Body 4" - parent_offset {{ x: 0.1 y: 0.1 }} - child_offset {{ x: -0.2 y: -0.2 }} - parent: "Aux 1" - child: "$ Body 4" - stiffness: 18000.0 - angular_damping: 20 - spring_damping: 80 - rotation: {{ z: 135 }} - angle_limit {{ - min: 30.0 - max: 70.0 - }} -}} -joints {{ - name: "$ Torso_Aux 2" - parent_offset {{ x: -0.2 y: 0.2 }} - child_offset {{ x: 0.1 y: -0.1 }} - parent: "$ Torso" - child: "Aux 2" - stiffness: 18000.0 - angular_damping: 20 - spring_damping: 80 - rotation {{ y: -90 }} - angle_limit {{ min: -30.0 max: 30.0 }} -}} -joints {{ - name: "Aux 2_$ Body 7" - parent_offset {{ x: -0.1 y: 0.1 }} - child_offset {{ x: 0.2 y: -0.2 }} - parent: "Aux 2" - child: "$ Body 7" - stiffness: 18000.0 - angular_damping: 20 - spring_damping: 80 - rotation {{ z: 45 }} - angle_limit {{ min: -70.0 max: -30.0 }} -}} -joints {{ - name: "$ Torso_Aux 3" - parent_offset {{ x: -0.2 y: -0.2 }} - child_offset {{ x: 0.1 y: 0.1 }} - parent: "$ Torso" - child: "Aux 3" - stiffness: 18000.0 - angular_damping: 20 - spring_damping: 80 - rotation {{ y: -90 }} - angle_limit {{ min: -30.0 max: 30.0 }} -}} -joints {{ - name: "Aux 3_$ Body 10" - parent_offset {{ x: -0.1 y: -0.1 }} - child_offset {{ - x: 0.2 - y: 0.2 - }} - parent: "Aux 3" - child: "$ Body 10" - stiffness: 18000.0 - angular_damping: 20 - spring_damping: 80 - rotation {{ z: 135 }} - angle_limit {{ min: -70.0 max: -30.0 }} -}} -joints {{ - name: "$ Torso_Aux 4" - parent_offset {{ x: 0.2 y: -0.2 }} - child_offset {{ x: -0.1 y: 0.1 }} - parent: "$ Torso" - child: "Aux 4" - stiffness: 18000.0 - angular_damping: 20 - spring_damping: 80 - rotation {{ y: -90 }} - angle_limit {{ min: -30.0 max: 30.0 }} -}} -joints {{ - name: "Aux 4_$ Body 13" - parent_offset {{ x: 0.1 y: -0.1 }} - child_offset {{ x: -0.2 y: 0.2 }} - parent: "Aux 4" - child: "$ Body 13" - stiffness: 18000.0 - angular_damping: 20 - spring_damping: 80 - rotation {{ z: 45 }} - angle_limit {{ min: 30.0 max: 70.0 }} -}} -actuators {{ - name: "$ Torso_Aux 1" - joint: "$ Torso_Aux 1" - strength: 350.0 - torque {{}} -}} -actuators {{ - name: "Aux 1_$ Body 4" - joint: "Aux 1_$ Body 4" - strength: 350.0 - torque {{}} -}} -actuators {{ - name: "$ Torso_Aux 2" - joint: "$ Torso_Aux 2" - strength: 350.0 - torque {{}} -}} -actuators {{ - name: "Aux 2_$ Body 7" - joint: "Aux 2_$ Body 7" - strength: 350.0 - torque {{}} -}} -actuators {{ - name: "$ Torso_Aux 3" - joint: "$ Torso_Aux 3" - strength: 350.0 - torque {{}} -}} -actuators {{ - name: "Aux 3_$ Body 10" - joint: "Aux 3_$ Body 10" - strength: 350.0 - torque {{}} -}} -actuators {{ - name: "$ Torso_Aux 4" - joint: "$ Torso_Aux 4" - strength: 350.0 - torque {{}} -}} -actuators {{ - name: "Aux 4_$ Body 13" - joint: "Aux 4_$ Body 13" - strength: 350.0 - torque {{}} -}} -friction: 1.0 -gravity {{ z: -9.8 }} -angular_damping: -0.05 -baumgarte_erp: 0.1 -collide_include {{ - first: "$ Torso" - second: "Ground" -}} -collide_include {{ - first: "$ Body 4" - second: "Ground" -}} -collide_include {{ - first: "$ Body 7" - second: "Ground" -}} -collide_include {{ - first: "$ Body 10" - second: "Ground" -}} -collide_include {{ - first: "$ Body 13" - second: "Ground" -}} -dt: {dt} -substeps: 10 -dynamics_mode: "legacy_spring" -""" - - -def create_modified_ant_env( - modify_dict: dict = {}, - episode_length: int = 1000, - action_repeat: int = 1, - auto_reset: bool = True, - batch_size: Optional[int] = None, - eval_metrics: bool = False, - **kwargs -): - """Creates a config modified Ant Env with a specified brax system.""" - default_settings = {"torso_mass": 15, "dt": 0.05} - for k, v in default_settings.items(): - if k not in modify_dict.keys(): - modify_dict[k] = v - config = _CONFIG_MODIFIED.format(**modify_dict) - - env = ModifiedAnt(config, **kwargs) - if episode_length is not None: - env = wrappers.EpisodeWrapper(env, episode_length, action_repeat) - if batch_size: - env = wrappers.VectorWrapper(env, batch_size) - if auto_reset: - env = wrappers.AutoResetWrapper(env) - if eval_metrics: - env = wrappers.EvalWrapper(env) - - return env diff --git a/evosax/problems/obs_norm.py b/evosax/problems/obs_norm.py deleted file mode 100644 index 2dd20df..0000000 --- a/evosax/problems/obs_norm.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2022 The EvoJAX Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Adapted from https://github.com/google/evojax/blob/main/evojax/obs_norm.py - -from typing import Tuple -from functools import partial - -import jax -import jax.numpy as jnp -import numpy as np - - -def normalize( - obs: jnp.ndarray, - obs_params: jnp.ndarray, - obs_shape: Tuple, - clip_value: float, - std_min_value: float, - std_max_value: float, -) -> jnp.ndarray: - """Normalize the given observation.""" - - obs_steps = obs_params[0] - running_mean, running_var = jnp.split(obs_params[1:], 2) - running_mean = running_mean.reshape(obs_shape) - running_var = running_var.reshape(obs_shape) - - variance = running_var / (obs_steps + 1.0) - variance = jnp.clip(variance, std_min_value, std_max_value) - return jnp.clip( - (obs - running_mean) / jnp.sqrt(variance), -clip_value, clip_value - ) - - -def update_obs_params( - obs_buffer: jnp.ndarray, obs_mask: jnp.ndarray, obs_params: jnp.ndarray -) -> jnp.ndarray: - """Update observation normalization parameters.""" - - obs_steps = obs_params[0] - running_mean, running_var = jnp.split(obs_params[1:], 2) - if obs_mask.ndim != obs_buffer.ndim: - obs_mask = obs_mask.reshape( - obs_mask.shape + (1,) * (obs_buffer.ndim - obs_mask.ndim) - ) - - new_steps = jnp.sum(obs_mask) - total_steps = obs_steps + new_steps - - input_to_old_mean = (obs_buffer - running_mean) * obs_mask - mean_diff = jnp.sum(input_to_old_mean / total_steps, axis=(0, 1)) - new_mean = running_mean + mean_diff - - input_to_new_mean = (obs_buffer - new_mean) * obs_mask - var_diff = jnp.sum(input_to_new_mean * input_to_old_mean, axis=(0, 1)) - new_var = running_var + var_diff - - return jnp.concatenate([jnp.ones(1) * total_steps, new_mean, new_var]) - - -class ObsNormalizer(object): - """Observation normalizer.""" - - def __init__( - self, - obs_shape: Tuple, - clip_value: float = 5.0, - std_min_value: float = 1e-6, - std_max_value: float = 1e6, - dummy: bool = False, - ): - """Initialization. - - Args: - obs_shape - Shape of the observations. - std_min_value - Minimum standard deviation. - std_max_value - Maximum standard deviation. - dummy - Whether this is a dummy normalizer. - """ - - self._obs_shape = obs_shape - self._obs_size = np.prod(obs_shape) - self._std_min_value = std_min_value - self._std_max_value = std_max_value - self._clip_value = clip_value - self.is_dummy = dummy - - @partial(jax.jit, static_argnums=(0,)) - def normalize_obs( - self, obs: jnp.ndarray, obs_params: jnp.ndarray - ) -> jnp.ndarray: - """Normalize the given observation. - - Args: - obs - The observation to be normalized. - Returns: - Normalized observation. - """ - - if self.is_dummy: - return obs - else: - return normalize( - obs=obs, - obs_params=obs_params, - obs_shape=self._obs_shape, - clip_value=self._clip_value, - std_min_value=self._std_min_value, - std_max_value=self._std_max_value, - ) - - @partial(jax.jit, static_argnums=(0,)) - def update_normalization_params( - self, - obs_buffer: jnp.ndarray, - obs_mask: jnp.ndarray, - obs_params: jnp.ndarray, - ) -> jnp.ndarray: - """Update internal parameters.""" - - if self.is_dummy: - return jnp.zeros_like(obs_params) - else: - return update_obs_params( - obs_buffer=obs_buffer, - obs_mask=obs_mask, - obs_params=obs_params, - ) - - @partial(jax.jit, static_argnums=(0,)) - def get_init_params(self) -> jnp.ndarray: - return jnp.zeros(1 + self._obs_size * 2) diff --git a/evosax/utils/evojax_wrapper.py b/evosax/utils/evojax_wrapper.py new file mode 100644 index 0000000..ae38b1f --- /dev/null +++ b/evosax/utils/evojax_wrapper.py @@ -0,0 +1,53 @@ +import chex +import jax +import jax.numpy as jnp +from evojax.algo.base import NEAlgorithm +from evosax import Strategy + + +class Evosax2JAX_Wrapper(NEAlgorithm): + """Wrapper for evosax-style ES for EvoJAX deployment.""" + + def __init__( + self, + evosax_strategy: Strategy, + param_size: int, + pop_size: int, + es_config: dict = {}, + es_params: dict = {}, + seed: int = 42, + ): + self.es = evosax_strategy( + popsize=pop_size, num_dims=param_size, maximize=True, **es_config + ) + self.es_params = self.es.default_params.replace(**es_params) + self.pop_size = pop_size + self.param_size = param_size + self.rand_key = jax.random.PRNGKey(seed=seed) + self.rand_key, init_key = jax.random.split(self.rand_key) + self.es_state = self.es.initialize(init_key, self.es_params) + + def ask(self) -> chex.Array: + """Ask strategy for next set of solution candidates to evaluate.""" + self.rand_key, ask_key = jax.random.split(self.rand_key) + self.params, self.es_state = self.es.ask( + ask_key, self.es_state, self.es_params + ) + return self.params + + def tell(self, fitness: chex.Array) -> None: + """Tell strategy about most recent fitness evaluations.""" + fit_re = self.fit_shaper.apply(self.params, fitness) + self.es_state = self.es.tell( + self.params, fit_re, self.es_state, self.es_params + ) + + @property + def best_params(self) -> chex.Array: + """Return set of mean/best parameters.""" + return jnp.array(self.es_state.mean, copy=True) + + @best_params.setter + def best_params(self, params: chex.Array) -> None: + """Update the best parameters stored internally.""" + self.es_state = self.es_state.replace(mean=jnp.array(params, copy=True)) diff --git a/evosax/utils/visualizer_2d.py b/evosax/utils/visualizer_2d.py new file mode 100644 index 0000000..2171213 --- /dev/null +++ b/evosax/utils/visualizer_2d.py @@ -0,0 +1,207 @@ +"""Fitness landscape visualizer and evaluation animator.""" +import chex +import jax.numpy as jnp +import numpy as np +import matplotlib.cm as cm +import matplotlib.pyplot as plt +import matplotlib.animation as animation +from evosax.problems.bbob import BBOB_fns, get_rotation + +cmap = cm.colors.LinearSegmentedColormap.from_list( + "Custom", [(0, "#2f9599"), (0.45, "#eee"), (1, "#8800ff")], N=256 +) + + +class BBOBVisualizer(object): + """Fitness landscape visualizer and evaluation animator.""" + + def __init__( + self, + X: chex.Array, + fn_name: str = "Rastrigin", + title: str = "", + use_3d: bool = False, + ): + self.X = X + self.title = title + self.fn_name = fn_name + self.use_3d = use_3d + if not self.use_3d: + self.fig, self.ax = plt.subplots(figsize=(6, 5)) + else: + self.fig = plt.figure(figsize=(6, 5)) + self.ax = self.fig.add_subplot(1, 1, 1, projection="3d") + self.fn_name = fn_name + self.fn = BBOB_fns[self.fn_name] + self.R = jnp.array(get_rotation(2, 0, b"R")) + self.Q = jnp.array(get_rotation(2, 0, b"Q")) + self.global_minima = [] + + self.x1_lower_bound, self.x1_upper_bound = -5, 5 + self.x2_lower_bound, self.x2_upper_bound = -5, 5 + + def animate(self, save_fname: str): + """Run animation for provided data.""" + ani = animation.FuncAnimation( + self.fig, + self.update, + frames=self.X.shape[0], + init_func=self.init, + blit=False, + interval=10, + ) + ani.save(save_fname) + + def init(self): + """Initialize the first frame for the animation.""" + if self.use_3d: + self.plot_contour_3d() + (self.scat,) = self.ax.plot( + self.X[0, :, 0], + self.X[0, :, 1], + jnp.ones(X.shape[1]) * 0.1, + marker="o", + c="r", + linestyle="", + markersize=3, + alpha=0.5, + ) + + else: + self.plot_contour_2d() + (self.scat,) = self.ax.plot( + self.X[0, :, 0], + self.X[0, :, 1], + marker="o", + c="r", + linestyle="", + markersize=3, + alpha=0.5, + ) + + return (self.scat,) + + def update(self, frame): + """Update the frame with the solutions evaluated in generation.""" + # Plot sample points + self.scat.set_data(self.X[frame, :, 0], self.X[frame, :, 1]) + if self.use_3d: + self.scat.set_3d_properties(jnp.ones(X.shape[1]) * 0.1) + self.ax.set_title( + f"{self.fn_name}: {self.title} - Generation {frame + 1}", + fontsize=15, + ) + self.fig.tight_layout() + return (self.scat,) + + def contour_function(self, x1, x2): + """Evaluate vmapped fitness landscape.""" + + def fn_val(x1, x2): + x = jnp.stack([x1, x2]) + return self.fn(x, self.R, self.Q) + + return jax.vmap(jax.vmap(fn_val, in_axes=(0, None)), in_axes=(None, 0))( + x1, x2 + ) + + def plot_contour_2d(self, save: bool = False): + """Plot 2d landscape contour.""" + + if save: + self.fig, self.ax = plt.subplots(figsize=(6, 5)) + self.ax.set_xlim(self.x1_lower_bound, self.x1_upper_bound) + self.ax.set_ylim(self.x2_lower_bound, self.x2_upper_bound) + self.ax.set_xlim(self.x1_lower_bound, self.x1_upper_bound) + self.ax.set_ylim(self.x2_lower_bound, self.x2_upper_bound) + + # Plot local minimum value + for m in self.global_minima: + self.ax.plot(m[0], m[1], "y*", ms=10) + self.ax.plot(m[0], m[1], "y*", ms=10) + + x1 = jnp.arange(self.x1_lower_bound, self.x1_upper_bound, 0.01) + x2 = jnp.arange(self.x2_lower_bound, self.x2_upper_bound, 0.01) + X, Y = np.meshgrid(x1, x2) + contour = self.contour_function(x1, x2) + self.ax.contour(X, Y, contour, levels=30, linewidths=0.5, colors="#999") + im = self.ax.contourf(X, Y, contour, levels=30, cmap=cmap, alpha=0.7) + self.ax.set_title(f"{self.fn_name} Function", fontsize=15) + self.ax.set_xlabel(r"$x_1$") + self.ax.set_ylabel(r"$x_2$") + self.fig.colorbar(im, ax=self.ax) + self.fig.tight_layout() + + if save: + plt.savefig(f"{self.fn_name}_2d.png", dpi=300) + + def plot_contour_3d(self, save: bool = False): + """Plot 3d landscape contour.""" + if save: + self.fig = plt.figure(figsize=(6, 5)) + self.ax = self.fig.add_subplot(1, 1, 1, projection="3d") + x1 = jnp.arange(self.x1_lower_bound, self.x1_upper_bound, 0.01) + x2 = jnp.arange(self.x2_lower_bound, self.x2_upper_bound, 0.01) + contour = self.contour_function(x1, x2) + X, Y = np.meshgrid(x1, x2) + self.ax.contour( + X, + Y, + contour, + zdir="z", + offset=np.min(contour), + levels=30, + cmap=cmap, + alpha=0.5, + ) + self.ax.plot_surface( + X, + Y, + contour, + cmap=cmap, + linewidth=0, + antialiased=True, + alpha=0.7, + ) + + # Rmove fills and set labels + self.ax.xaxis.pane.fill = False + self.ax.yaxis.pane.fill = False + self.ax.zaxis.pane.fill = False + + self.ax.xaxis.set_tick_params(labelsize=8) + self.ax.yaxis.set_tick_params(labelsize=8) + self.ax.zaxis.set_tick_params(labelsize=8) + + self.ax.set_xlabel(r"$x_1$") + self.ax.set_ylabel(r"$x_2$") + self.ax.set_zlabel(r"$f(x)$") + self.ax.set_title(f"{self.fn_name} Function", fontsize=15) + self.fig.tight_layout() + if save: + plt.savefig(f"{self.fn_name}_3d.png", dpi=300) + + +if __name__ == "__main__": + import jax + from jax.config import config + + config.update("jax_enable_x64", True) + + rng = jax.random.PRNGKey(42) + + for fn_name in [ + "BuecheRastrigin", + ]: # BBOB_fns.keys(): + print(f"Start 2d/3d - {fn_name}") + visualizer = BBOBVisualizer(None, fn_name, "") + visualizer.plot_contour_2d(save=True) + visualizer.plot_contour_3d(save=True) + + # # Test animations + # # All solutions from single run (10 gens, 16 pmembers, 2 dims) + # X = jax.random.normal(rng, shape=(10, 16, 2)) + # visualizer = BBOBVisualizer(X, "Ackley", "Test Strategy", use_3d=True) + # visualizer.animate("Ackley_3d.gif") + # visualizer = BBOBVisualizer(X, "Ackley", "Test Strategy", use_3d=False) + # visualizer.animate("Ackley_2d.gif") From 246255ebca098af0efdab63b5b17d40154d1d292 Mon Sep 17 00:00:00 2001 From: RobertTLange Date: Mon, 21 Nov 2022 14:04:30 +0100 Subject: [PATCH 06/13] Add GuidedES & Adan optimizer --- .gitignore | 1 + CHANGELOG.md | 20 +++- evosax/__init__.py | 3 + evosax/strategies/__init__.py | 2 + evosax/strategies/ars.py | 2 +- evosax/strategies/esmc.py | 2 +- evosax/strategies/guided_es.py | 168 +++++++++++++++++++++++++++++ evosax/strategies/open_es.py | 2 +- evosax/strategies/persistent_es.py | 2 +- evosax/strategies/pgpe.py | 10 +- evosax/utils/__init__.py | 4 +- evosax/utils/evojax_wrapper.py | 9 +- evosax/utils/optimizer.py | 56 ++++++++++ tests/conftest.py | 50 +++++++-- tests/test_fitness_rollout.py | 56 +++------- tests/test_strategy_api.py | 4 +- tests/test_strategy_run.py | 6 +- 17 files changed, 327 insertions(+), 70 deletions(-) create mode 100644 evosax/strategies/guided_es.py diff --git a/.gitignore b/.gitignore index 1aa7a32..873359c 100755 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +asebo.py des.py bbob.py # Standard ROB excludes diff --git a/CHANGELOG.md b/CHANGELOG.md index f896be6..d41def2 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,24 +1,38 @@ ### Work-in-Progress -- [ ] Make xNES work with all optimizers (currently only GD) - Implement more strategies - [ ] Large-scale CMA-ES variants - [ ] [LM-CMA](https://www.researchgate.net/publication/282612269_LM-CMA_An_alternative_to_L-BFGS_for_large-scale_black_Box_optimization) - [ ] [VkD-CMA](https://hal.inria.fr/hal-01306551v1/document), [Code](https://gist.github.com/youheiakimoto/2fb26c0ace43c22b8f19c7796e69e108) - - [ ] [sNES](https://www.jmlr.org/papers/volume15/wierstra14a/wierstra14a.pdf) (separable version of xNES) - [ ] [ASEBO](https://proceedings.neurips.cc/paper/2019/file/88bade49e98db8790df275fcebb37a13-Paper.pdf) - [ ] [RBO](http://proceedings.mlr.press/v100/choromanski20a/choromanski20a.pdf) - Encoding methods - via special reshape wrappers - [ ] Discrete Cosine Transform - [ ] Wavelet Based Encoding (van Steenkiste, 2016) - - [ ] Hypernetworks (Ha - start with simple MLP) + - [ ] CNN Hypernetwork (Ha - start with simple MLP) ### [v0.1.0] - [TBD] ##### Added - Adds a `total_env_steps` counter to both `GymFitness` and `BraxFitness` for easier sample efficiency comparability with RL algorithms. +- Support for new strategies/genetic algorithms + - SAMR-GA (Clune et al., 2008) + - GESMR-GA (Kumar et al., 2022) + - SNES (Wierstra et al., 2014) + - DES (Lange et al., 2022) + - Guided ES (Maheswaranathan et al., 2018) +- Adds full set of BBOB low-dimensional functions (`BBOBFitness`) +- Adds 2D visualizer animating sampled points (`BBOBVisualizer`) +- Adds `Evosax2JAXWrapper` to wrap all evosax strategies +- Adds Adan optimizer (Xie et al., 2022) + +##### Changed + +- `ParameterReshaper` can now be directly applied from within the strategy. You simply have to provide a `pholder_params` pytree at strategy instantiation (and no `num_dims`). +- `FitnessShaper` can also be directly applied from within the strategy. This makes it easier to track the best performing member across generations and addresses issue #32. Simply provide the fitness shaping settings as args to the strategy (`maximize`, `centered_rank`, ...) +- Removes Brax fitness (use EvoJAX version instead) ##### Fixed diff --git a/evosax/__init__.py b/evosax/__init__.py index 716691c..7103c5e 100755 --- a/evosax/__init__.py +++ b/evosax/__init__.py @@ -26,6 +26,7 @@ DES, SAMR_GA, GESMR_GA, + GuidedES, ) from .utils import FitnessShaper, ParameterReshaper, ESLog from .networks import NetworkMapper @@ -59,6 +60,7 @@ "DES": DES, "SAMR_GA": SAMR_GA, "GESMR_GA": GESMR_GA, + "GuidedES": GuidedES, } __all__ = [ @@ -97,4 +99,5 @@ "DES", "SAMR_GA", "GESMR_GA", + "GuidedES", ] diff --git a/evosax/strategies/__init__.py b/evosax/strategies/__init__.py index 64e5036..1dda633 100755 --- a/evosax/strategies/__init__.py +++ b/evosax/strategies/__init__.py @@ -24,6 +24,7 @@ from .des import DES from .samr_ga import SAMR_GA from .gesmr_ga import GESMR_GA +from .guided_es import GuidedES __all__ = [ @@ -53,4 +54,5 @@ "DES", "SAMR_GA", "GESMR_GA", + "GuidedES", ] diff --git a/evosax/strategies/ars.py b/evosax/strategies/ars.py index c9b8fd7..e4b58db 100644 --- a/evosax/strategies/ars.py +++ b/evosax/strategies/ars.py @@ -48,7 +48,7 @@ def __init__( assert 0 <= elite_ratio <= 1 self.elite_ratio = elite_ratio self.elite_popsize = max(1, int(self.popsize / 2 * self.elite_ratio)) - assert opt_name in ["sgd", "adam", "rmsprop", "clipup"] + assert opt_name in ["sgd", "adam", "rmsprop", "clipup", "adan"] self.optimizer = GradientOptimizer[opt_name](self.num_dims) self.strategy_name = "ARS" diff --git a/evosax/strategies/esmc.py b/evosax/strategies/esmc.py index b7dc356..30fe820 100644 --- a/evosax/strategies/esmc.py +++ b/evosax/strategies/esmc.py @@ -45,7 +45,7 @@ def __init__( """ super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) assert self.popsize & 1, "Population size must be odd" - assert opt_name in ["sgd", "adam", "rmsprop", "clipup"] + assert opt_name in ["sgd", "adam", "rmsprop", "clipup", "adan"] self.optimizer = GradientOptimizer[opt_name](self.num_dims) self.strategy_name = "ESMC" diff --git a/evosax/strategies/guided_es.py b/evosax/strategies/guided_es.py new file mode 100644 index 0000000..d5962bb --- /dev/null +++ b/evosax/strategies/guided_es.py @@ -0,0 +1,168 @@ +import jax +import jax.numpy as jnp +import chex +from typing import Tuple, Optional, Union +from functools import partial +from ..strategy import Strategy +from ..utils import GradientOptimizer, OptState, OptParams +from flax import struct +from evosax.utils import get_best_fitness_member + + +@struct.dataclass +class EvoState: + mean: chex.Array + sigma: float + opt_state: OptState + grad_subspace: chex.Array + best_member: chex.Array + best_fitness: float = jnp.finfo(jnp.float32).max + gen_counter: int = 0 + + +@struct.dataclass +class EvoParams: + opt_params: OptParams + sigma_init: float = 0.03 + sigma_decay: float = 1.0 + sigma_limit: float = 0.01 + alpha: float = 0.5 + beta: float = 1.0 + init_min: float = 0.0 + init_max: float = 0.0 + clip_min: float = -jnp.finfo(jnp.float32).max + clip_max: float = jnp.finfo(jnp.float32).max + + +class GuidedES(Strategy): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + opt_name: str = "sgd", + subspace_dims: int = 1, # k param in example notebook + **fitness_kwargs: Union[bool, int, float] + ): + """Guided ES (Maheswaranathan et al., 2018) + Reference: https://arxiv.org/abs/1806.10230 + Note that there are a couple of JAX-based adaptations: + """ + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) + assert not self.popsize & 1, "Population size must be even" + assert opt_name in ["sgd", "adam", "rmsprop", "clipup", "adan"] + assert ( + subspace_dims <= self.num_dims + ), "Subspace has to be smaller than optimization dims." + self.optimizer = GradientOptimizer[opt_name](self.num_dims) + self.subspace_dims = subspace_dims + self.strategy_name = "GuidedES" + + @property + def params_strategy(self) -> EvoParams: + """Return default parameters of evolution strategy.""" + return EvoParams(opt_params=self.optimizer.default_params) + + def initialize_strategy( + self, rng: chex.PRNGKey, params: EvoParams + ) -> EvoState: + """`initialize` the evolution strategy.""" + rng_init, rng_sub = jax.random.split(rng) + initialization = jax.random.uniform( + rng_init, + (self.num_dims,), + minval=params.init_min, + maxval=params.init_max, + ) + + grad_subspace = jax.random.normal( + rng_sub, (self.subspace_dims, self.num_dims) + ) + + state = EvoState( + mean=initialization, + sigma=params.sigma_init, + opt_state=self.optimizer.initialize(params.opt_params), + grad_subspace=grad_subspace, + best_member=initialization, + ) + return state + + def ask_strategy( + self, rng: chex.PRNGKey, state: EvoState, params: EvoParams + ) -> Tuple[chex.Array, EvoState]: + """`ask` for new parameter candidates to evaluate next.""" + a = state.sigma * jnp.sqrt(params.alpha / self.num_dims) + c = state.sigma * jnp.sqrt((1.0 - params.alpha) / self.subspace_dims) + key_full, key_sub = jax.random.split(rng, 2) + eps_full = jax.random.normal( + key_full, shape=(self.num_dims, int(self.popsize / 2)) + ) + eps_subspace = jax.random.normal( + key_sub, shape=(self.subspace_dims, int(self.popsize / 2)) + ) + Q, _ = jnp.linalg.qr(state.grad_subspace) + # Antithetic sampling of noise + z_plus = a * eps_full + c * jnp.dot(Q, eps_subspace) + z_plus = jnp.swapaxes(z_plus, 0, 1) + z = jnp.concatenate([z_plus, -1.0 * z_plus]) + x = state.mean + z + return x, state + + @partial(jax.jit, static_argnums=(0,)) + def tell( + self, + x: chex.Array, + fitness: chex.Array, + state: EvoState, + params: Optional[EvoParams] = None, + gradient: Optional[chex.Array] = None, + ) -> EvoState: + """`tell` performance data for strategy state update.""" + # Use default hyperparameters if no other settings provided + if params is None: + params = self.default_params + + # Flatten params if using param reshaper for ES update + if self.use_param_reshaper: + x = self.param_reshaper.flatten(x) + + # Perform fitness reshaping inside of strategy tell call (if desired) + fitness_re = self.fitness_shaper.apply(x, fitness) + + # Reconstruct noise from last mean/std estimates + noise = (x - state.mean) / state.sigma + noise_1 = noise[: int(self.popsize / 2)] + fit_1 = fitness_re[: int(self.popsize / 2)] + fit_2 = fitness_re[int(self.popsize / 2) :] + fit_diff = fit_1 - fit_2 + fit_diff_noise = jnp.dot(noise_1.T, fit_diff) + theta_grad = (params.beta / self.popsize) * fit_diff_noise + + # Add grad FIFO-style to subspace archive (only if provided else FD) + grad_subspace = jnp.zeros((self.subspace_dims, self.num_dims)) + grad_subspace = grad_subspace.at[:-1, :].set(state.grad_subspace[1:, :]) + if gradient is not None: + grad_subspace = grad_subspace.at[-1, :].set(gradient) + else: + grad_subspace = grad_subspace.at[-1, :].set(theta_grad) + state = state.replace(grad_subspace=grad_subspace) + + # Grad update using optimizer instance - decay lrate if desired + mean, opt_state = self.optimizer.step( + state.mean, theta_grad, state.opt_state, params.opt_params + ) + opt_state = self.optimizer.update(opt_state, params.opt_params) + + # Update lrate and standard deviation based on min and decay + sigma = state.sigma * params.sigma_decay + sigma = jnp.maximum(sigma, params.sigma_limit) + state = state.replace(mean=mean, sigma=sigma, opt_state=opt_state) + + # Check if there is a new best member & update trackers + best_member, best_fitness = get_best_fitness_member(x, fitness, state) + return state.replace( + best_member=best_member, + best_fitness=best_fitness, + gen_counter=state.gen_counter + 1, + ) diff --git a/evosax/strategies/open_es.py b/evosax/strategies/open_es.py index 72bba86..8dff784 100755 --- a/evosax/strategies/open_es.py +++ b/evosax/strategies/open_es.py @@ -43,7 +43,7 @@ def __init__( Inspired by: https://github.com/hardmaru/estool/blob/master/es.py""" super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) assert not self.popsize & 1, "Population size must be even" - assert opt_name in ["sgd", "adam", "rmsprop", "clipup"] + assert opt_name in ["sgd", "adam", "rmsprop", "clipup", "adan"] self.optimizer = GradientOptimizer[opt_name](self.num_dims) self.strategy_name = "OpenES" diff --git a/evosax/strategies/persistent_es.py b/evosax/strategies/persistent_es.py index 2aa2cde..118df7f 100644 --- a/evosax/strategies/persistent_es.py +++ b/evosax/strategies/persistent_es.py @@ -48,7 +48,7 @@ def __init__( """ super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) assert not self.popsize & 1, "Population size must be even" - assert opt_name in ["sgd", "adam", "rmsprop", "clipup"] + assert opt_name in ["sgd", "adam", "rmsprop", "clipup", "adan"] self.optimizer = GradientOptimizer[opt_name](self.num_dims) self.strategy_name = "PersistentES" diff --git a/evosax/strategies/pgpe.py b/evosax/strategies/pgpe.py index 8da27a7..217316b 100755 --- a/evosax/strategies/pgpe.py +++ b/evosax/strategies/pgpe.py @@ -50,7 +50,7 @@ def __init__( self.elite_popsize = max(1, int(self.popsize / 2 * self.elite_ratio)) assert not self.popsize & 1, "Population size must be even" - assert opt_name in ["sgd", "adam", "rmsprop", "clipup"] + assert opt_name in ["sgd", "adam", "rmsprop", "clipup", "adan"] self.optimizer = GradientOptimizer[opt_name](self.num_dims) self.strategy_name = "PGPE" @@ -87,7 +87,7 @@ def ask_strategy( (int(self.popsize / 2), self.num_dims), ) z = jnp.concatenate([z_plus, -1.0 * z_plus]) - x = state.mean + z * state.sigma.reshape(1, self.num_dims) + x = state.mean + state.sigma * z return x, state def tell_strategy( @@ -100,9 +100,9 @@ def tell_strategy( """Update both mean and dim.-wise isotropic Gaussian scale.""" # Reconstruct noise from last mean/std estimates noise = (x - state.mean) / state.sigma - noise_1 = noise[::2] - fit_1 = fitness[::2] - fit_2 = fitness[1::2] + noise_1 = noise[: int(self.popsize / 2)] + fit_1 = fitness[: int(self.popsize / 2)] + fit_2 = fitness[int(self.popsize / 2) :] elite_idx = jnp.minimum(fit_1, fit_2).argsort()[: self.elite_popsize] fitness_elite = jnp.concatenate([fit_1[elite_idx], fit_2[elite_idx]]) diff --git a/evosax/utils/__init__.py b/evosax/utils/__init__.py index e3af896..c93ee75 100755 --- a/evosax/utils/__init__.py +++ b/evosax/utils/__init__.py @@ -11,13 +11,14 @@ from .helpers import get_best_fitness_member # Import Gradient Based Optimizer step functions -from .optimizer import SGD, Adam, RMSProp, ClipUp, OptState, OptParams +from .optimizer import SGD, Adam, RMSProp, ClipUp, Adan, OptState, OptParams GradientOptimizer = { "sgd": SGD, "adam": Adam, "rmsprop": RMSProp, "clipup": ClipUp, + "adan": Adan, } @@ -31,6 +32,7 @@ "Adam", "RMSProp", "ClipUp", + "Adan", "OptState", "OptParams", ] diff --git a/evosax/utils/evojax_wrapper.py b/evosax/utils/evojax_wrapper.py index ae38b1f..7f36efb 100644 --- a/evosax/utils/evojax_wrapper.py +++ b/evosax/utils/evojax_wrapper.py @@ -15,12 +15,16 @@ def __init__( pop_size: int, es_config: dict = {}, es_params: dict = {}, + opt_params: dict = {}, seed: int = 42, ): self.es = evosax_strategy( - popsize=pop_size, num_dims=param_size, maximize=True, **es_config + popsize=pop_size, num_dims=param_size, **es_config ) self.es_params = self.es.default_params.replace(**es_params) + if len(opt_params.keys()) > 0: + opt_params = self.es_params.opt_params.replace(**opt_params) + self.es_params = self.es_params.replace(opt_params=opt_params) self.pop_size = pop_size self.param_size = param_size self.rand_key = jax.random.PRNGKey(seed=seed) @@ -37,9 +41,8 @@ def ask(self) -> chex.Array: def tell(self, fitness: chex.Array) -> None: """Tell strategy about most recent fitness evaluations.""" - fit_re = self.fit_shaper.apply(self.params, fitness) self.es_state = self.es.tell( - self.params, fit_re, self.es_state, self.es_params + self.params, fitness, self.es_state, self.es_params ) @property diff --git a/evosax/utils/optimizer.py b/evosax/utils/optimizer.py index 02e46de..fd25d32 100644 --- a/evosax/utils/optimizer.py +++ b/evosax/utils/optimizer.py @@ -16,6 +16,8 @@ class OptState: lrate: float m: chex.Array v: Optional[chex.Array] = None + n: Optional[chex.Array] = None + last_grads: Optional[chex.Array] = None gen_counter: int = 0 @@ -27,6 +29,7 @@ class OptParams: momentum: Optional[float] = None beta_1: Optional[float] = None beta_2: Optional[float] = None + beta_3: Optional[float] = None eps: Optional[float] = None max_speed: Optional[float] = None @@ -241,3 +244,56 @@ def clip(velocity: chex.Array, max_speed: float): m = clip(velocity, params.max_speed) mean_new = mean - state.lrate * m return mean_new, state.replace(m=m) + + +class Adan(Optimizer): + def __init__(self, num_dims: int): + """JAX-Compatible Adan Optimizer (Xi et al., 2022) + Reference: https://arxiv.org/pdf/2208.06677.pdf""" + super().__init__(num_dims) + self.opt_name = "adan" + + @property + def params_opt(self) -> Dict[str, float]: + """Return default Adam parameters.""" + return { + "beta_1": 0.98, + "beta_2": 0.92, + "beta_3": 0.99, + "eps": 1e-8, + } + + def initialize_opt(self, params: OptParams) -> OptState: + """Initialize the m, v, n trace of the optimizer.""" + return OptState( + m=jnp.zeros(self.num_dims), + v=jnp.zeros(self.num_dims), + n=jnp.zeros(self.num_dims), + last_grads=jnp.zeros(self.num_dims), + lrate=params.lrate_init, + ) + + def step_opt( + self, + mean: chex.Array, + grads: chex.Array, + state: OptState, + params: OptParams, + ) -> Tuple[chex.Array, OptState]: + """Perform a simple Adan GD step.""" + m = (1 - params.beta_1) * grads + params.beta_1 * state.m + grad_diff = grads - state.last_grads + v = (1 - params.beta_2) * grad_diff + params.beta_2 * state.v + n = (1 - params.beta_3) * ( + grads + params.beta_2 * grad_diff + ) ** 2 + params.beta_3 * state.n + + mhat = m / (1 - params.beta_1 ** (state.gen_counter + 1)) + vhat = v / (1 - params.beta_2 ** (state.gen_counter + 1)) + nhat = n / (1 - params.beta_3 ** (state.gen_counter + 1)) + mean_new = mean - state.lrate * (mhat + params.beta_2 * vhat) / ( + jnp.sqrt(nhat) + params.eps + ) + return mean_new, state.replace( + m=m, v=v, n=n, last_grads=grads, gen_counter=state.gen_counter + 1 + ) diff --git a/tests/conftest.py b/tests/conftest.py index 6436556..8ac2335 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,6 +28,10 @@ def pytest_generate_tests(metafunc): "xNES", "SNES", "ESMC", + "DES", + "SAMR_GA", + # "GESMR_GA", + "GuidedES", ], ) else: @@ -38,24 +42,50 @@ def pytest_generate_tests(metafunc): metafunc.parametrize( "classic_name", [ - "rosenbrock", - "quadratic", - "ackley", - "griewank", - "rastrigin", - "schwefel", - "himmelblau", - "six-hump", + "Sphere", + "EllipsoidalOriginal", + "RastriginOriginal", + "BuecheRastrigin", + "LinearSlope", + # Part 2: Functions with low or moderate conditions + "AttractiveSector", + "StepEllipsoidal", + "RosenbrockOriginal", + "RosenbrockRotated", + # Part 3: Functions with high conditioning and unimodal + "EllipsoidalRotated", + "Discus", + "BentCigar", + "SharpRidge", + "DifferentPowers", + # Part 4: Multi-modal functions with adequate global structure + "RastriginRotated", + "Weierstrass", + "SchaffersF7", + "SchaffersF7IllConditioned", + "GriewankRosenbrock", + # Part 5: Multi-modal functions with weak global structure + "Schwefel", + "Lunacek", + "Gallagher101Me", + "Gallagher21Hi", + # "Katsuura", + # Part 6: Additional low-d functions (not in BBOB) + "Linear", + "Ackley", + "DixonPrice", ], ) else: - metafunc.parametrize("classic_name", ["rosenbrock"]) + metafunc.parametrize("classic_name", ["Sphere"]) if "env_name" in metafunc.fixturenames: if metafunc.config.getoption("all"): metafunc.parametrize( "env_name", - ["CartPole-v1", "ant"], + [ + "CartPole-v1", + ], ) else: metafunc.parametrize("env_name", ["CartPole-v1"]) diff --git a/tests/test_fitness_rollout.py b/tests/test_fitness_rollout.py index 2f35564..f4a7438 100644 --- a/tests/test_fitness_rollout.py +++ b/tests/test_fitness_rollout.py @@ -2,9 +2,8 @@ import jax.numpy as jnp from evosax import CMA_ES, ARS, ParameterReshaper, NetworkMapper from evosax.problems import ( - ClassicFitness, + BBOBFitness, GymFitness, - BraxFitness, VisionFitness, SequenceFitness, ) @@ -12,9 +11,7 @@ def test_classic_rollout(classic_name: str): rng = jax.random.PRNGKey(0) - evaluator = ClassicFitness( - classic_name, num_dims=2, num_rollouts=2, noise_std=0.1 - ) + evaluator = BBOBFitness(classic_name, num_dims=2) strategy = CMA_ES(popsize=20, num_dims=2, elite_ratio=0.5) params = strategy.default_params state = strategy.initialize(rng, params) @@ -23,29 +20,19 @@ def test_classic_rollout(classic_name: str): rng, rng_gen, rng_eval = jax.random.split(rng, 3) x, state = strategy.ask(rng_gen, state, params) fitness = evaluator.rollout(rng_eval, x) - assert fitness.shape == (20, 2) + assert fitness.shape == (20,) def test_env_ffw_rollout(env_name: str): rng = jax.random.PRNGKey(0) - if env_name in ["CartPole-v1"]: - evaluator = GymFitness(env_name, num_env_steps=100, num_rollouts=10) - network = NetworkMapper["MLP"]( - num_hidden_units=64, - num_hidden_layers=2, - num_output_units=evaluator.action_shape, - hidden_activation="relu", - output_activation="categorical", - ) - else: - evaluator = BraxFitness(env_name, num_env_steps=100, num_rollouts=10) - network = NetworkMapper["MLP"]( - num_hidden_units=64, - num_hidden_layers=2, - num_output_units=evaluator.action_shape, - hidden_activation="tanh", - output_activation="tanh", - ) + evaluator = GymFitness(env_name, num_env_steps=100, num_rollouts=10) + network = NetworkMapper["MLP"]( + num_hidden_units=64, + num_hidden_layers=2, + num_output_units=evaluator.action_shape, + hidden_activation="relu", + output_activation="categorical", + ) pholder = jnp.zeros((1, evaluator.input_shape[0])) net_params = network.init( rng, @@ -69,21 +56,12 @@ def test_env_ffw_rollout(env_name: str): def test_env_rec_rollout(env_name: str): rng = jax.random.PRNGKey(0) - if env_name in ["CartPole-v1"]: - evaluator = GymFitness(env_name, num_env_steps=100, num_rollouts=10) - network = NetworkMapper["LSTM"]( - num_hidden_units=64, - num_output_units=evaluator.action_shape, - output_activation="categorical", - ) - - else: - evaluator = BraxFitness(env_name, num_env_steps=100, num_rollouts=10) - network = NetworkMapper["LSTM"]( - num_hidden_units=64, - num_output_units=evaluator.action_shape, - output_activation="tanh", - ) + evaluator = GymFitness(env_name, num_env_steps=100, num_rollouts=10) + network = NetworkMapper["LSTM"]( + num_hidden_units=64, + num_output_units=evaluator.action_shape, + output_activation="categorical", + ) pholder = jnp.zeros((1, evaluator.input_shape[0])) carry_init = network.initialize_carry() diff --git a/tests/test_strategy_api.py b/tests/test_strategy_api.py index 81812af..3d7318e 100644 --- a/tests/test_strategy_api.py +++ b/tests/test_strategy_api.py @@ -1,6 +1,6 @@ import jax from evosax import Strategies -from evosax.problems import ClassicFitness +from evosax.problems import BBOBFitness def test_strategy_ask(strategy_name): @@ -30,7 +30,7 @@ def test_strategy_ask_tell(strategy_name): params = strategy.default_params state = strategy.initialize(rng, params) x, state = strategy.ask(rng, state, params) - evaluator = ClassicFitness("rosenbrock", num_dims=2) + evaluator = BBOBFitness("Sphere", num_dims=2) fitness = evaluator.rollout(rng, x) state = strategy.tell(x, fitness, state, params) return diff --git a/tests/test_strategy_run.py b/tests/test_strategy_run.py index 07eedf2..bdacaf2 100644 --- a/tests/test_strategy_run.py +++ b/tests/test_strategy_run.py @@ -1,7 +1,7 @@ import jax import jax.numpy as jnp from evosax import Strategies -from evosax.problems import ClassicFitness +from evosax.problems import BBOBFitness from evosax.utils import FitnessShaper from functools import partial @@ -17,7 +17,7 @@ def test_strategy_run(strategy_name): popsize = 21 else: popsize = 20 - evaluator = ClassicFitness("rosenbrock", 2) + evaluator = BBOBFitness("Sphere", 2) fitness_shaper = FitnessShaper() batch_eval = evaluator.rollout @@ -46,7 +46,7 @@ def test_strategy_scan(strategy_name): popsize = 21 else: popsize = 20 - evaluator = ClassicFitness("rosenbrock", 2) + evaluator = BBOBFitness("Sphere", 2) fitness_shaper = FitnessShaper() batch_eval = evaluator.rollout From 55cffe17c5e7498e532d5c30cb7b3c55a0ce30a8 Mon Sep 17 00:00:00 2001 From: RobertTLange Date: Mon, 21 Nov 2022 19:20:55 +0100 Subject: [PATCH 07/13] ASEBO --- .gitignore | 1 - CHANGELOG.md | 2 + README.md | 2 + evosax/__init__.py | 3 + evosax/strategies/__init__.py | 2 + evosax/strategies/asebo.py | 168 ++++++++++++++++++++++++++++++++++ evosax/strategies/pgpe.py | 6 +- tests/conftest.py | 1 + 8 files changed, 181 insertions(+), 4 deletions(-) create mode 100644 evosax/strategies/asebo.py diff --git a/.gitignore b/.gitignore index 873359c..1aa7a32 100755 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -asebo.py des.py bbob.py # Standard ROB excludes diff --git a/CHANGELOG.md b/CHANGELOG.md index d41def2..0258fd6 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ - SNES (Wierstra et al., 2014) - DES (Lange et al., 2022) - Guided ES (Maheswaranathan et al., 2018) + - ASEBO (Choromanski et al., 2019) - Adds full set of BBOB low-dimensional functions (`BBOBFitness`) - Adds 2D visualizer animating sampled points (`BBOBVisualizer`) - Adds `Evosax2JAXWrapper` to wrap all evosax strategies @@ -37,6 +38,7 @@ ##### Fixed - Fixed reward masking in `GymFitness`. Using `jnp.sum(dones) >= 1` for cumulative return computation zeros out the final timestep, which is wrong. That's why there were problems with sparse reward gym environments (e.g. Mountain Car). +- Fixed PGPE sample indexing. ### [v0.0.9] - 15/06/2022 diff --git a/README.md b/README.md index c841768..ca3f0c8 100755 --- a/README.md +++ b/README.md @@ -59,6 +59,8 @@ state.best_member, state.best_fitness | DES | [Lange et al. (2022)]() | [`DES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/des.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | SAMR-GA | [Clune et al. (2008)](https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1000187) | [`SAMR_GA`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/samr_ga.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | GESMR-GA | [Kumar et al. (2022)](https://arxiv.org/abs/2204.04817) | [`GESMR_GA`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/gesmr_ga.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| Guided ES | [Maheswaranathan et al. (2018)](https://arxiv.org/abs/1806.10230) | [`GuidedES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/guided_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| ASEBO | [Choromanski et al. (2019)](https://arxiv.org/abs/1903.04268) | [`GuidedES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/asebo.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) diff --git a/evosax/__init__.py b/evosax/__init__.py index 7103c5e..ef27916 100755 --- a/evosax/__init__.py +++ b/evosax/__init__.py @@ -27,6 +27,7 @@ SAMR_GA, GESMR_GA, GuidedES, + ASEBO, ) from .utils import FitnessShaper, ParameterReshaper, ESLog from .networks import NetworkMapper @@ -61,6 +62,7 @@ "SAMR_GA": SAMR_GA, "GESMR_GA": GESMR_GA, "GuidedES": GuidedES, + "ASEBO": ASEBO, } __all__ = [ @@ -100,4 +102,5 @@ "SAMR_GA", "GESMR_GA", "GuidedES", + "ASEBO", ] diff --git a/evosax/strategies/__init__.py b/evosax/strategies/__init__.py index 1dda633..d6fed99 100755 --- a/evosax/strategies/__init__.py +++ b/evosax/strategies/__init__.py @@ -25,6 +25,7 @@ from .samr_ga import SAMR_GA from .gesmr_ga import GESMR_GA from .guided_es import GuidedES +from .asebo import ASEBO __all__ = [ @@ -55,4 +56,5 @@ "SAMR_GA", "GESMR_GA", "GuidedES", + "ASEBO", ] diff --git a/evosax/strategies/asebo.py b/evosax/strategies/asebo.py new file mode 100644 index 0000000..44fe70f --- /dev/null +++ b/evosax/strategies/asebo.py @@ -0,0 +1,168 @@ +import jax +import jax.numpy as jnp +import chex +from typing import Tuple, Optional, Union +from ..strategy import Strategy +from ..utils import GradientOptimizer, OptState, OptParams +from flax import struct + + +@struct.dataclass +class EvoState: + mean: chex.Array + sigma: float + opt_state: OptState + grad_subspace: chex.Array + alpha: float + UUT: chex.Array + UUT_ort: chex.Array + best_member: chex.Array + best_fitness: float = jnp.finfo(jnp.float32).max + gen_counter: int = 0 + + +@struct.dataclass +class EvoParams: + opt_params: OptParams + sigma_init: float = 0.03 + sigma_decay: float = 1.0 + sigma_limit: float = 0.01 + grad_decay: float = 0.99 + init_min: float = 0.0 + init_max: float = 0.0 + clip_min: float = -jnp.finfo(jnp.float32).max + clip_max: float = jnp.finfo(jnp.float32).max + + +class ASEBO(Strategy): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + subspace_dims: int = 2, + opt_name: str = "adam", + **fitness_kwargs: Union[bool, int, float] + ): + """ASEBO (Choromanski et al., 2019) + Reference: https://arxiv.org/abs/1903.04268 + Note that there are a couple of JAX-based adaptations: + 1. We always sample a fixed population size per generation + 2. We keep a fixed archive of gradients to estimate the subspace + """ + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) + assert not self.popsize & 1, "Population size must be even" + assert opt_name in ["sgd", "adam", "rmsprop", "clipup"] + assert ( + subspace_dims <= self.num_dims + ), "Subspace has to be smaller than optimization dims." + self.optimizer = GradientOptimizer[opt_name](self.num_dims) + self.subspace_dims = subspace_dims + self.strategy_name = "ASEBO" + + @property + def params_strategy(self) -> EvoParams: + """Return default parameters of evolution strategy.""" + return EvoParams(opt_params=self.optimizer.default_params) + + def initialize_strategy( + self, rng: chex.PRNGKey, params: EvoParams + ) -> EvoState: + """`initialize` the evolution strategy.""" + initialization = jax.random.uniform( + rng, + (self.num_dims,), + minval=params.init_min, + maxval=params.init_max, + ) + + grad_subspace = jnp.zeros((self.subspace_dims, self.num_dims)) + + state = EvoState( + mean=initialization, + sigma=params.sigma_init, + opt_state=self.optimizer.initialize(params.opt_params), + grad_subspace=grad_subspace, + alpha=1.0, + UUT=jnp.zeros((self.num_dims, self.num_dims)), + UUT_ort=jnp.zeros((self.num_dims, self.num_dims)), + best_member=initialization, + ) + return state + + def ask_strategy( + self, rng: chex.PRNGKey, state: EvoState, params: EvoParams + ) -> Tuple[chex.Array, EvoState]: + """`ask` for new parameter candidates to evaluate next.""" + # Antithetic sampling of noise + X = state.grad_subspace + X -= jnp.mean(X, axis=0) + U, S, Vt = jnp.linalg.svd(X, full_matrices=False) + + def svd_flip(u, v): + # columns of u, rows of v + max_abs_cols = jnp.argmax(jnp.abs(u), axis=0) + signs = jnp.sign(u[max_abs_cols, jnp.arange(u.shape[1])]) + u *= signs + v *= signs[:, jnp.newaxis] + return u, v + + U, Vt = svd_flip(U, Vt) + U = Vt[: int(self.popsize / 2)] + UUT = jnp.matmul(U.T, U) + + U_ort = Vt[int(self.popsize / 2) :] + UUT_ort = jnp.matmul(U_ort.T, U_ort) + cov = ( + state.sigma * (state.alpha / self.num_dims) * jnp.eye(self.num_dims) + + ((1 - state.alpha) / int(self.popsize / 2)) * UUT + ) + chol = jnp.linalg.cholesky(cov) + noise = jax.random.normal(rng, (self.num_dims, int(self.popsize / 2))) + z_plus = jnp.swapaxes(chol @ noise, 0, 1) + z_plus /= jnp.linalg.norm(z_plus, axis=-1)[:, jnp.newaxis] + z = jnp.concatenate([z_plus, -1.0 * z_plus]) + x = state.mean + z + return x, state.replace(UUT=UUT, UUT_ort=UUT_ort) + + def tell_strategy( + self, + x: chex.Array, + fitness: chex.Array, + state: EvoState, + params: EvoParams, + ) -> EvoState: + """`tell` performance data for strategy state update.""" + # Reconstruct noise from last mean/std estimates + noise = (x - state.mean) / state.sigma + noise_1 = noise[: int(self.popsize / 2)] + fit_1 = fitness[: int(self.popsize / 2)] + fit_2 = fitness[int(self.popsize / 2) :] + fit_diff_noise = jnp.dot(noise_1.T, fit_1 - fit_2) + theta_grad = 1.0 / 2.0 * fit_diff_noise + + alpha = jnp.linalg.norm( + jnp.dot(theta_grad, state.UUT_ort) + ) / jnp.linalg.norm(jnp.dot(theta_grad, state.UUT)) + + # Add grad FIFO-style to subspace archive (only if provided else FD) + grad_subspace = jnp.zeros((self.subspace_dims, self.num_dims)) + grad_subspace = grad_subspace.at[:-1, :].set(state.grad_subspace[1:, :]) + grad_subspace = grad_subspace.at[-1, :].set(theta_grad) + state = state.replace(grad_subspace=grad_subspace) + + # Normalize gradients by norm / num_dims + theta_grad /= jnp.linalg.norm(theta_grad) / self.num_dims + 1e-8 + + # Grad update using optimizer instance - decay lrate if desired + mean, opt_state = self.optimizer.step( + state.mean, theta_grad, state.opt_state, params.opt_params + ) + opt_state = self.optimizer.update(opt_state, params.opt_params) + + # Update lrate and standard deviation based on min and decay + sigma = state.sigma * params.sigma_decay + sigma = jnp.maximum(sigma, params.sigma_limit) + return state.replace( + mean=mean, sigma=sigma, opt_state=opt_state, alpha=alpha + ) diff --git a/evosax/strategies/pgpe.py b/evosax/strategies/pgpe.py index 217316b..bc1ea4e 100755 --- a/evosax/strategies/pgpe.py +++ b/evosax/strategies/pgpe.py @@ -100,9 +100,9 @@ def tell_strategy( """Update both mean and dim.-wise isotropic Gaussian scale.""" # Reconstruct noise from last mean/std estimates noise = (x - state.mean) / state.sigma - noise_1 = noise[: int(self.popsize / 2)] - fit_1 = fitness[: int(self.popsize / 2)] - fit_2 = fitness[int(self.popsize / 2) :] + noise_1 = noise[::2] + fit_1 = fitness[::2] + fit_2 = fitness[1::2] elite_idx = jnp.minimum(fit_1, fit_2).argsort()[: self.elite_popsize] fitness_elite = jnp.concatenate([fit_1[elite_idx], fit_2[elite_idx]]) diff --git a/tests/conftest.py b/tests/conftest.py index 8ac2335..49a755f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,6 +32,7 @@ def pytest_generate_tests(metafunc): "SAMR_GA", # "GESMR_GA", "GuidedES", + "ASEBO", ], ) else: From d532b8660920affe4e17533bd536b96d87dd01a5 Mon Sep 17 00:00:00 2001 From: RobertTLange Date: Wed, 23 Nov 2022 12:35:47 +0100 Subject: [PATCH 08/13] Add CR-FM-NES & DES --- .gitignore | 1 - CHANGELOG.md | 2 +- README.md | 155 ++++++++--------- evosax/__init__.py | 3 + evosax/strategies/__init__.py | 2 + evosax/strategies/cr_fm_nes.py | 295 +++++++++++++++++++++++++++++++++ evosax/strategies/des.py | 107 ++++++++++++ evosax/strategies/xnes.py | 2 +- 8 files changed, 487 insertions(+), 80 deletions(-) create mode 100644 evosax/strategies/cr_fm_nes.py create mode 100644 evosax/strategies/des.py diff --git a/.gitignore b/.gitignore index 1aa7a32..225d5f0 100755 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -des.py bbob.py # Standard ROB excludes .sync-config.cson diff --git a/CHANGELOG.md b/CHANGELOG.md index 0258fd6..119099a 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,6 @@ - [ ] Large-scale CMA-ES variants - [ ] [LM-CMA](https://www.researchgate.net/publication/282612269_LM-CMA_An_alternative_to_L-BFGS_for_large-scale_black_Box_optimization) - [ ] [VkD-CMA](https://hal.inria.fr/hal-01306551v1/document), [Code](https://gist.github.com/youheiakimoto/2fb26c0ace43c22b8f19c7796e69e108) - - [ ] [ASEBO](https://proceedings.neurips.cc/paper/2019/file/88bade49e98db8790df275fcebb37a13-Paper.pdf) - [ ] [RBO](http://proceedings.mlr.press/v100/choromanski20a/choromanski20a.pdf) - Encoding methods - via special reshape wrappers @@ -24,6 +23,7 @@ - DES (Lange et al., 2022) - Guided ES (Maheswaranathan et al., 2018) - ASEBO (Choromanski et al., 2019) + - CR-FM-NES (Nomura & Ono, 2022) - Adds full set of BBOB low-dimensional functions (`BBOBFitness`) - Adds 2D visualizer animating sampled points (`BBOBVisualizer`) - Adds `Evosax2JAXWrapper` to wrap all evosax strategies diff --git a/README.md b/README.md index ca3f0c8..d8dc3c2 100755 --- a/README.md +++ b/README.md @@ -56,12 +56,12 @@ state.best_member, state.best_fitness | GLD | [Golovin et al. (2019)](https://arxiv.org/pdf/1911.06317.pdf) | [`GLD`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/gld.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | Simulated Annealing | [Rasdi Rere et al. (2015)](https://www.sciencedirect.com/science/article/pii/S1877050915035759) | [`SimAnneal`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/sim_anneal.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | ESMC | [Merchant et al. (2021)](https://proceedings.mlr.press/v139/merchant21a.html) | [`ESMC`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/esmc.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) -| DES | [Lange et al. (2022)]() | [`DES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/des.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| DES | [Lange et al. (2022)](https://arxiv.org/abs/2211.11260) | [`DES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/des.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | SAMR-GA | [Clune et al. (2008)](https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1000187) | [`SAMR_GA`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/samr_ga.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | GESMR-GA | [Kumar et al. (2022)](https://arxiv.org/abs/2204.04817) | [`GESMR_GA`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/gesmr_ga.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | Guided ES | [Maheswaranathan et al. (2018)](https://arxiv.org/abs/1806.10230) | [`GuidedES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/guided_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | ASEBO | [Choromanski et al. (2019)](https://arxiv.org/abs/1903.04268) | [`GuidedES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/asebo.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) - +| CR-FM-NES | [Nomura & Ono (2022)](https://arxiv.org/abs/2201.11422) | [`CR-FM-NES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/cr_fm_nes.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) ## Installation ⏳ @@ -87,12 +87,12 @@ In order to use JAX on your accelerators, you can find more details in the [JAX * 📓 [LRateTune-PES](https://github.com/RobertTLange/evosax/blob/main/examples/04_lrate_pes.ipynb): Persistent ES on meta-learning problem as in [Vicol et al. (2021)](http://proceedings.mlr.press/v139/vicol21a.html). * 📓 [Quadratic-PBT](https://github.com/RobertTLange/evosax/blob/main/examples/05_quadratic_pbt.ipynb): PBT on toy quadratic problem as in [Jaderberg et al. (2017)](https://arxiv.org/abs/1711.09846). * 📓 [Restart-Wrappers](https://github.com/RobertTLange/evosax/blob/main/examples/06_restart_es.ipynb): Custom restart wrappers as e.g. used in (B)IPOP-CMA-ES. -* 📓 [Brax Control](https://github.com/RobertTLange/evosax/blob/main/examples/07_brax_control.ipynb): Evolve Tanh MLPs on Brax tasks using the `evosax` wrapper. +* 📓 [Brax Control](https://github.com/RobertTLange/evosax/blob/main/examples/07_brax_control.ipynb): Evolve Tanh MLPs on Brax tasks using the `EvoJAX` wrapper. * 📓 [Indirect Encodings](https://github.com/RobertTLange/evosax/blob/main/examples/08_encodings.ipynb): Find out how many parameters we need to evolve a pendulum controller. -## Key Selling Points 💵 +## Key Features 💵 -- **Strategy Diversity**: `evosax` implements more than 10 classical and modern neuroevolution strategies. All of them follow the same simple `ask`/`eval` API and come with tailored tools such as the [ClipUp](https://arxiv.org/abs/2008.02387) optimizer, parameter reshaping into PyTrees and fitness shaping (see below). +- **Strategy Diversity**: `evosax` implements more than 30 classical and modern neuroevolution strategies. All of them follow the same simple `ask`/`eval` API and come with tailored tools such as the [ClipUp](https://arxiv.org/abs/2008.02387) optimizer, parameter reshaping into PyTrees and fitness shaping (see below). - **Vectorization/Parallelization of `ask`/`tell` Calls**: Both `ask` and `tell` calls can leverage `jit`, `vmap`/`pmap`. This enables vectorized/parallel rollouts of different evolution strategies. @@ -176,78 +176,79 @@ fit_shaper = FitnessShaper(centered_rank=True, # Shape the evaluated fitness scores fit_shaped = fit_shaper.apply(x, fitness) ``` - -- **Strategy Restart Wrappers**: You can also choose from a set of different restart mechanisms, which will relaunch a strategy (with e.g. new population size) based on termination criteria. Note: For all restart strategies which alter the population size the ask and tell methods will have to be re-compiled at the time of change. Note that all strategies can also be executed without explicitly providing `es_params`. In this case the default parameters will be used. - -```Python -from evosax import CMA_ES -from evosax.restarts import BIPOP_Restarter - -# Define a termination criterion (kwargs - fitness, state, params) -def std_criterion(fitness, state, params): - """Restart strategy if fitness std across population is small.""" - return fitness.std() < 0.001 - -# Instantiate Base CMA-ES & wrap with BIPOP restarts -# Pass strategy-specific kwargs separately (e.g. elite_ration or opt_name) -strategy = CMA_ES(num_dims, popsize, elite_ratio) -re_strategy = BIPOP_Restarter( - strategy, - stop_criteria=[std_criterion], - strategy_kwargs={"elite_ratio": elite_ratio} - ) -state = re_strategy.initialize(rng) - -# ask/tell loop - restarts are automatically handled -rng, rng_gen, rng_eval = jax.random.split(rng, 3) -x, state = re_strategy.ask(rng_gen, state) -fitness = ... # Your population evaluation fct -state = re_strategy.tell(x, fitness, state) -``` - -- **Batch Strategy Rollouts**: *Work-in-progress*. We are currently also working on different ways of incorporating multiple subpopulations with different communication protocols. - -```Python -from evosax.experimental.subpops import BatchStrategy - -# Instantiates 5 CMA-ES subpops of 20 members -strategy = BatchStrategy( - strategy_name="CMA_ES", - num_dims=4096, - popsize=100, - num_subpops=5, - strategy_kwargs={"elite_ratio": 0.5}, - communication="best_subpop", - ) - -state = strategy.initialize(rng) -# Ask for evaluation candidates of different subpopulation ES -x, state = strategy.ask(rng_iter, state) -fitness = ... -state = strategy.tell(x, fitness, state) -``` - -- **Indirect Encodings**: *Work-in-progress*. ES can struggle with high-dimensional search spaces (e.g. due to harder estimation of covariances). One potential way to alleviate this challenge, is to use indirect parameter encodings in a lower dimensional space. So far we provide JAX-compatible encodings with random projections (Gaussian/Rademacher) and Hypernetworks for MLPs. They act as drop-in replacements for the `ParameterReshaper`: - -```Python -from evosax.experimental.decodings import RandomDecoder, HyperDecoder - -# For arbitrary network architectures / search spaces -num_encoding_dims = 6 -param_reshaper = RandomDecoder(num_encoding_dims, net_params) -x_shaped = param_reshaper.reshape(x) - -# For MLP-based models we also support a HyperNetwork en/decoding -reshaper = HyperDecoder( - net_params, - hypernet_config={ - "num_latent_units": 3, # Latent units per module kernel/bias - "num_hidden_units": 2, # Hidden dimensionality of a_i^j embedding - }, - ) -x_shaped = param_reshaper.reshape(x) -``` - +
+ Additonal Work-In-Progress + **Strategy Restart Wrappers**: *Work-in-progress*. You can also choose from a set of different restart mechanisms, which will relaunch a strategy (with e.g. new population size) based on termination criteria. Note: For all restart strategies which alter the population size the ask and tell methods will have to be re-compiled at the time of change. Note that all strategies can also be executed without explicitly providing `es_params`. In this case the default parameters will be used. + + ```Python + from evosax import CMA_ES + from evosax.restarts import BIPOP_Restarter + + # Define a termination criterion (kwargs - fitness, state, params) + def std_criterion(fitness, state, params): + """Restart strategy if fitness std across population is small.""" + return fitness.std() < 0.001 + + # Instantiate Base CMA-ES & wrap with BIPOP restarts + # Pass strategy-specific kwargs separately (e.g. elite_ration or opt_name) + strategy = CMA_ES(num_dims, popsize, elite_ratio) + re_strategy = BIPOP_Restarter( + strategy, + stop_criteria=[std_criterion], + strategy_kwargs={"elite_ratio": elite_ratio} + ) + state = re_strategy.initialize(rng) + + # ask/tell loop - restarts are automatically handled + rng, rng_gen, rng_eval = jax.random.split(rng, 3) + x, state = re_strategy.ask(rng_gen, state) + fitness = ... # Your population evaluation fct + state = re_strategy.tell(x, fitness, state) + ``` + + - **Batch Strategy Rollouts**: *Work-in-progress*. We are currently also working on different ways of incorporating multiple subpopulations with different communication protocols. + + ```Python + from evosax.experimental.subpops import BatchStrategy + + # Instantiates 5 CMA-ES subpops of 20 members + strategy = BatchStrategy( + strategy_name="CMA_ES", + num_dims=4096, + popsize=100, + num_subpops=5, + strategy_kwargs={"elite_ratio": 0.5}, + communication="best_subpop", + ) + + state = strategy.initialize(rng) + # Ask for evaluation candidates of different subpopulation ES + x, state = strategy.ask(rng_iter, state) + fitness = ... + state = strategy.tell(x, fitness, state) + ``` + + - **Indirect Encodings**: *Work-in-progress*. ES can struggle with high-dimensional search spaces (e.g. due to harder estimation of covariances). One potential way to alleviate this challenge, is to use indirect parameter encodings in a lower dimensional space. So far we provide JAX-compatible encodings with random projections (Gaussian/Rademacher) and Hypernetworks for MLPs. They act as drop-in replacements for the `ParameterReshaper`: + + ```Python + from evosax.experimental.decodings import RandomDecoder, HyperDecoder + + # For arbitrary network architectures / search spaces + num_encoding_dims = 6 + param_reshaper = RandomDecoder(num_encoding_dims, net_params) + x_shaped = param_reshaper.reshape(x) + + # For MLP-based models we also support a HyperNetwork en/decoding + reshaper = HyperDecoder( + net_params, + hypernet_config={ + "num_latent_units": 3, # Latent units per module kernel/bias + "num_hidden_units": 2, # Hidden dimensionality of a_i^j embedding + }, + ) + x_shaped = param_reshaper.reshape(x) + ``` +
## Resources & Other Great JAX-ES Tools 📝 diff --git a/evosax/__init__.py b/evosax/__init__.py index ef27916..59ca684 100755 --- a/evosax/__init__.py +++ b/evosax/__init__.py @@ -28,6 +28,7 @@ GESMR_GA, GuidedES, ASEBO, + CR_FM_NES, ) from .utils import FitnessShaper, ParameterReshaper, ESLog from .networks import NetworkMapper @@ -63,6 +64,7 @@ "GESMR_GA": GESMR_GA, "GuidedES": GuidedES, "ASEBO": ASEBO, + "CR_FM_NES": CR_FM_NES, } __all__ = [ @@ -103,4 +105,5 @@ "GESMR_GA", "GuidedES", "ASEBO", + "CR_FM_NES", ] diff --git a/evosax/strategies/__init__.py b/evosax/strategies/__init__.py index d6fed99..15bcda3 100755 --- a/evosax/strategies/__init__.py +++ b/evosax/strategies/__init__.py @@ -26,6 +26,7 @@ from .gesmr_ga import GESMR_GA from .guided_es import GuidedES from .asebo import ASEBO +from .cr_fm_nes import CR_FM_NES __all__ = [ @@ -57,4 +58,5 @@ "GESMR_GA", "GuidedES", "ASEBO", + "CR_FM_NES", ] diff --git a/evosax/strategies/cr_fm_nes.py b/evosax/strategies/cr_fm_nes.py new file mode 100644 index 0000000..44a0634 --- /dev/null +++ b/evosax/strategies/cr_fm_nes.py @@ -0,0 +1,295 @@ +import jax +import jax.numpy as jnp +import chex +import math +from typing import Tuple, Optional, Union +from ..strategy import Strategy +from flax import struct + + +@struct.dataclass +class EvoState: + mean: chex.Array + sigma: float + v: chex.Array + D: chex.Array + p_sigma: chex.Array + p_c: chex.Array + w_rank_hat: chex.Array + w_rank: chex.Array + z: chex.Array + y: chex.Array + best_member: chex.Array + best_fitness: float = jnp.finfo(jnp.float32).max + gen_counter: int = 0 + + +@struct.dataclass +class EvoParams: + mu_eff: float + c_s: float + c_c: float + c1: float + chi_N: float + h_inv: float + alpha_dist: float + lrate_mean: float = 1.0 + lrate_move_sigma: float = 0.1 + lrate_stag_sigma: float = 0.1 + lrate_conv_sigma: float = 0.1 + lrate_B: float = 0.1 + sigma_init: float = 1.0 + init_min: float = 0.0 + init_max: float = 0.0 + clip_min: float = -jnp.finfo(jnp.float32).max + clip_max: float = jnp.finfo(jnp.float32).max + + +def get_recombination_weights(popsize: int) -> Tuple[chex.Array, chex.Array]: + """Get recombination weights for different ranks.""" + + def get_weight(i): + return jnp.log(popsize / 2 + 1) - jnp.log(i) + + w_rank_hat = jax.vmap(get_weight)(jnp.arange(1, popsize + 1)) + w_rank_hat = w_rank_hat * (w_rank_hat >= 0) + w_rank = w_rank_hat / sum(w_rank_hat) - (1.0 / popsize) + return w_rank_hat.reshape(-1, 1), w_rank.reshape(-1, 1) + + +def get_h_inv(dim: int) -> float: + dim = min(dim, 2000) + f = lambda a: ((1.0 + a * a) * math.exp(a * a / 2.0) / 0.24) - 10.0 - dim + f_prime = lambda a: (1.0 / 0.24) * a * math.exp(a * a / 2.0) * (3.0 + a * a) + h_inv = 1.0 + counter = 0 + while abs(f(h_inv)) > 1e-10: + counter += 1 + h_inv = h_inv - 0.5 * (f(h_inv) / f_prime(h_inv)) + return h_inv + + +def w_dist_hat(alpha_dist: float, z: chex.Array) -> chex.Array: + return jnp.exp(alpha_dist * jnp.linalg.norm(z)) + + +class CR_FM_NES(Strategy): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + **fitness_kwargs: Union[bool, int, float] + ): + """Cost-Reduced Fast-Moving Natural ES (Nomura & Ono, 2022) + Reference: https://arxiv.org/abs/2201.11422 + """ + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) + assert not self.popsize & 1, "Population size must be even" + self.strategy_name = "CR_FM_NES" + + @property + def default_params(self) -> EvoParams: + """Return default parameters of evolutionary strategy.""" + w_rank_hat, w_rank = get_recombination_weights(self.popsize) + mueff = 1 / ( + (w_rank + (1 / self.popsize)).T @ (w_rank + (1 / self.popsize)) + ) + c_s = (mueff + 2.0) / (self.num_dims + mueff + 5.0) + c_c = (4.0 + mueff / self.num_dims) / ( + self.num_dims + 4.0 + 2.0 * mueff / self.num_dims + ) + c1_cma = 2.0 / (jnp.power(self.num_dims + 1.3, 2) + mueff) + chi_N = jnp.sqrt(self.num_dims) * ( + 1.0 + - 1.0 / (4.0 * self.num_dims) + + 1.0 / (21.0 * self.num_dims * self.num_dims) + ) + h_inv = get_h_inv(self.num_dims) + alpha_dist = h_inv * jnp.minimum( + 1.0, jnp.sqrt(self.popsize / self.num_dims) + ) + lrate_move_sigma = 1.0 + lrate_stag_sigma = jnp.tanh( + (0.024 * self.popsize + 0.7 * self.num_dims + 20.0) + / (self.num_dims + 12.0) + ) + lrate_conv_sigma = 2.0 * jnp.tanh( + (0.025 * self.popsize + 0.75 * self.num_dims + 10.0) + / (self.num_dims + 4.0) + ) + c1 = c1_cma * (self.num_dims - 5) / 6 + lrate_B = jnp.tanh( + (jnp.minimum(0.02 * self.popsize, 3 * jnp.log(self.num_dims)) + 5) + / (0.23 * self.num_dims + 25) + ) + params = EvoParams( + lrate_move_sigma=lrate_move_sigma, + lrate_stag_sigma=lrate_stag_sigma, + lrate_conv_sigma=lrate_conv_sigma, + lrate_B=lrate_B, + mu_eff=mueff, + c_s=c_s, + c_c=c_c, + c1=c1, + chi_N=chi_N, + alpha_dist=alpha_dist, + h_inv=h_inv, + ) + return params + + def initialize_strategy( + self, rng: chex.PRNGKey, params: EvoParams + ) -> EvoState: + """`initialize` the evolutionary strategy.""" + rng_init, rng_v = jax.random.split(rng) + initialization = jax.random.uniform( + rng_init, + (self.num_dims,), + minval=params.init_min, + maxval=params.init_max, + ) + w_rank_hat, w_rank = get_recombination_weights(self.popsize) + state = EvoState( + mean=initialization, + sigma=params.sigma_init, + v=jax.random.normal(rng_v, shape=(self.num_dims, 1)) + / jnp.sqrt(self.num_dims), + D=jnp.ones([self.num_dims, 1]), + p_sigma=jnp.zeros((self.num_dims, 1)), + p_c=jnp.zeros((self.num_dims, 1)), + z=jnp.zeros((self.popsize, self.num_dims)), + y=jnp.zeros((self.popsize, self.num_dims)), + w_rank_hat=w_rank_hat.reshape(-1, 1), + w_rank=w_rank, + best_member=initialization, + ) + + return state + + def ask_strategy( + self, rng: chex.PRNGKey, state: EvoState, params: EvoParams + ) -> Tuple[chex.Array, EvoState]: + """`ask` for new parameter candidates to evaluate next.""" + z_plus = jax.random.normal( + rng, + (int(self.popsize / 2), self.num_dims), + ) + z = jnp.concatenate([z_plus, -1.0 * z_plus]) + z = jnp.swapaxes(z, 0, 1) + normv = jnp.linalg.norm(state.v) + normv2 = normv ** 2 + vbar = state.v / normv + + # Rescale/reparametrize noise + y = z + (jnp.sqrt(1 + normv2) - 1) * vbar @ (vbar.T @ z) + x = state.mean[:, None] + state.sigma * y * state.D + x = jnp.swapaxes(x, 0, 1) + return x, state.replace(z=z, y=y) + + def tell_strategy( + self, + x: chex.Array, + fitness: chex.Array, + state: EvoState, + params: EvoParams, + ) -> EvoState: + """`tell` performance data for strategy state update.""" + ranks = fitness.argsort() + z = state.z[:, ranks] + y = state.y[:, ranks] + x = jnp.swapaxes(x, 0, 1)[:, ranks] + + # Update evolution path p_sigma + p_sigma = (1 - params.c_s) * state.p_sigma + jnp.sqrt( + params.c_s * (2.0 - params.c_s) * params.mu_eff + ) * (z @ state.w_rank) + p_sigma_norm = jnp.linalg.norm(p_sigma) + + # Calculate distance weight + w_tmp = state.w_rank_hat * jax.vmap(w_dist_hat, in_axes=(None, 1))( + params.alpha_dist, z + ).reshape(-1, 1) + weights_dist = w_tmp / sum(w_tmp) - 1.0 / self.popsize + + # switching weights and learning rate + p_sigma_cond = p_sigma_norm >= params.chi_N + weights = jax.lax.select(p_sigma_cond, weights_dist, state.w_rank) + lrate_sigma = jax.lax.select( + p_sigma_cond, params.lrate_move_sigma, params.lrate_stag_sigma + ) + lrate_sigma = jax.lax.select( + p_sigma_norm >= 0.1 * params.chi_N, + lrate_sigma, + params.lrate_conv_sigma, + ) + + # update evolution path p_c and mean + wxm = (x - state.mean[:, None]) @ weights + p_c = (1.0 - params.c_c) * state.p_c + jnp.sqrt( + params.c_c * (2.0 - params.c_c) * params.mu_eff + ) * wxm / state.sigma + mean = state.mean + params.lrate_mean * wxm.squeeze() + + normv = jnp.linalg.norm(state.v) + vbar = state.v / normv + normv2 = normv ** 2 + normv4 = normv2 ** 2 + + exY = jnp.append(y, p_c / state.D, axis=1) + yy = exY * exY + ip_yvbar = vbar.T @ exY + yvbar = exY * vbar + gammav = 1.0 + normv2 + vbarbar = vbar * vbar + alphavd = jnp.minimum( + 1, + jnp.sqrt( + normv4 + (2 * gammav - jnp.sqrt(gammav)) / jnp.max(vbarbar) + ) + / (2 + normv2), + ) + t = exY * ip_yvbar - vbar * (ip_yvbar ** 2 + gammav) / 2 + b = -(1 - alphavd ** 2) * normv4 / gammav + 2 * alphavd ** 2 + H = jnp.ones([self.num_dims, 1]) * 2 - (b + 2 * alphavd ** 2) * vbarbar + invH = H ** (-1) + s_step1 = ( + yy + - normv2 / gammav * (yvbar * ip_yvbar) + - jnp.ones([self.num_dims, self.popsize + 1]) + ) + ip_vbart = vbar.T @ t + s_step2 = s_step1 - alphavd / gammav * ( + (2 + normv2) * (t * vbar) - normv2 * vbarbar @ ip_vbart + ) + invHvbarbar = invH * vbarbar + ip_s_step2invHvbarbar = invHvbarbar.T @ s_step2 + s = (s_step2 * invH) - b / ( + 1 + b * vbarbar.T @ invHvbarbar + ) * invHvbarbar @ ip_s_step2invHvbarbar + ip_svbarbar = vbarbar.T @ s + t = t - alphavd * ((2 + normv2) * (s * vbar) - vbar @ ip_svbarbar) + + # update v, D covariance ingredients + exw = jnp.append( + params.lrate_B * weights, + jnp.array([params.c1]).reshape(1, 1), + axis=0, + ) + v = state.v + (t @ exw) / normv + D = state.D + (s @ exw) * state.D + # calculate detA + nthrootdetA = jnp.exp( + jnp.sum(jnp.log(D)) / self.num_dims + + jnp.log(1 + v.T @ v) / (2 * self.num_dims) + )[0][0] + D = D / nthrootdetA + # update sigma + G_s = ( + jnp.sum((z * z - jnp.ones([self.num_dims, self.popsize])) @ weights) + / self.num_dims + ) + sigma = state.sigma * jnp.exp(lrate_sigma / 2 * G_s) + return state.replace( + p_sigma=p_sigma, mean=mean, p_c=p_c, v=v, D=D, sigma=sigma + ) diff --git a/evosax/strategies/des.py b/evosax/strategies/des.py new file mode 100644 index 0000000..58dd273 --- /dev/null +++ b/evosax/strategies/des.py @@ -0,0 +1,107 @@ +import jax +import jax.numpy as jnp +import chex +from typing import Tuple, Optional, Union +from ..strategy import Strategy +from flax import struct +from flax import linen as nn + + +@struct.dataclass +class EvoState: + mean: chex.Array + sigma: chex.Array + weights: chex.Array # Weights for population members + best_member: chex.Array + best_fitness: float = jnp.finfo(jnp.float32).max + gen_counter: int = 0 + + +@struct.dataclass +class EvoParams: + temperature: float = 12.5 # Temperature for softmax weights + lrate_sigma: float = 0.1 # Learning rate for population std + lrate_mean: float = 1.0 # Learning rate for population mean + sigma_init: float = 1.0 # Standard deviation + init_min: float = 0.0 + init_max: float = 0.0 + clip_min: float = -jnp.finfo(jnp.float32).max + clip_max: float = jnp.finfo(jnp.float32).max + + +def get_des_weights(popsize: int, temperature: float = 12.5): + """Compute discovered recombination weights.""" + ranks = jnp.arange(popsize) + ranks /= ranks.size - 1 + ranks = ranks - 0.5 + sigout = nn.sigmoid(temperature * ranks) + weights = nn.softmax(-20 * sigout) + return weights + + +class DES(Strategy): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + ): + """Discovered Evolution Strategy (Lange et al., 2022)""" + super().__init__(popsize, num_dims, pholder_params) + self.strategy_name = "DES" + + @property + def params_strategy(self) -> EvoParams: + """Return default parameters of evolution strategy.""" + # Only parents have positive weight - equal weighting! + return EvoParams() + + def initialize_strategy( + self, rng: chex.PRNGKey, params: EvoParams + ) -> EvoState: + """`initialize` the evolution strategy.""" + weights = get_des_weights(self.popsize, params.temperature) + initialization = jax.random.uniform( + rng, + (self.num_dims,), + minval=params.init_min, + maxval=params.init_max, + ) + state = EvoState( + mean=initialization, + sigma=params.sigma_init * jnp.ones(self.num_dims), + weights=weights.reshape(-1, 1), + best_member=initialization, + ) + return state + + def ask_strategy( + self, rng: chex.PRNGKey, state: EvoState, params: EvoParams + ) -> Tuple[chex.Array, EvoState]: + """`ask` for new proposed candidates to evaluate next.""" + z = jax.random.normal(rng, (self.popsize, self.num_dims)) # ~ N(0, I) + x = state.mean + z * state.sigma.reshape( + 1, self.num_dims + ) # ~ N(m, σ^2 I) + return x, state + + def tell_strategy( + self, + x: chex.Array, + fitness: chex.Array, + state: EvoState, + params: EvoParams, + ) -> EvoState: + """`tell` update to ES state.""" + weights = state.weights + x = x[fitness.argsort()] + # Weighted updates + weighted_mean = (weights * x).sum(axis=0) + weighted_sigma = jnp.sqrt( + (weights * (x - state.mean) ** 2).sum(axis=0) + 1e-06 + ) + mean = state.mean + params.lrate_mean * (weighted_mean - state.mean) + sigma = state.sigma + params.lrate_sigma * ( + weighted_sigma - state.sigma + ) + return state.replace(mean=mean, sigma=sigma) diff --git a/evosax/strategies/xnes.py b/evosax/strategies/xnes.py index b934524..d4d31f6 100644 --- a/evosax/strategies/xnes.py +++ b/evosax/strategies/xnes.py @@ -91,7 +91,7 @@ def ask_strategy( noise = jax.random.normal(rng, (self.popsize, self.num_dims)) def scale_orient(n, sigma, B): - return state.sigma * state.B.T @ n + return sigma * B.T @ n scaled_noise = jax.vmap(scale_orient, in_axes=(0, None, None))( noise, state.sigma, state.B From 552cf0f13ab1a0c079cd798e68ca42c3e3536e86 Mon Sep 17 00:00:00 2001 From: RobertTLange Date: Fri, 25 Nov 2022 20:04:31 +0100 Subject: [PATCH 09/13] Add sigma/lrate params to ES init --- evosax/strategies/ars.py | 33 ++++++++++++++---- evosax/strategies/asebo.py | 52 ++++++++++++++++++++++++----- evosax/strategies/bipop_cma_es.py | 4 ++- evosax/strategies/cma_es.py | 5 +++ evosax/strategies/cr_fm_nes.py | 5 +++ evosax/strategies/esmc.py | 28 ++++++++++++++-- evosax/strategies/full_iamalgam.py | 23 ++++++++++--- evosax/strategies/gld.py | 2 +- evosax/strategies/guided_es.py | 30 +++++++++++++---- evosax/strategies/indep_iamalgam.py | 21 +++++++++--- evosax/strategies/ipop_cma_es.py | 2 ++ evosax/strategies/lm_ma_es.py | 5 +++ evosax/strategies/ma_es.py | 5 +++ evosax/strategies/open_es.py | 31 ++++++++++++++--- evosax/strategies/persistent_es.py | 31 ++++++++++++++--- evosax/strategies/pgpe.py | 31 ++++++++++++++--- evosax/strategies/rm_es.py | 5 +++ evosax/strategies/sep_cma_es.py | 5 +++ evosax/strategies/sim_anneal.py | 14 +++++++- evosax/strategies/simple_es.py | 6 +++- evosax/strategies/simple_ga.py | 23 +++++++++---- evosax/strategies/snes.py | 6 +++- evosax/strategies/xnes.py | 9 ++++- evosax/strategy.py | 8 +++++ evosax/utils/__init__.py | 12 ++++++- evosax/utils/helpers.py | 1 + evosax/utils/optimizer.py | 14 ++++++-- evosax/utils/reshape_fitness.py | 20 ++++++----- evosax/utils/reshape_params.py | 14 +++++++- 29 files changed, 375 insertions(+), 70 deletions(-) diff --git a/evosax/strategies/ars.py b/evosax/strategies/ars.py index e4b58db..0273474 100644 --- a/evosax/strategies/ars.py +++ b/evosax/strategies/ars.py @@ -3,7 +3,7 @@ import chex from typing import Tuple, Optional, Union from ..strategy import Strategy -from ..utils import GradientOptimizer, OptState, OptParams +from ..utils import GradientOptimizer, OptState, OptParams, exp_decay from flax import struct @@ -37,6 +37,12 @@ def __init__( pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.1, opt_name: str = "sgd", + lrate_init: float = 0.05, + lrate_decay: float = 1.0, + lrate_limit: float = 0.001, + sigma_init: float = 0.03, + sigma_decay: float = 1.0, + sigma_limit: float = 0.01, **fitness_kwargs: Union[bool, int, float] ): """Augmented Random Search (Mania et al., 2018) @@ -52,10 +58,28 @@ def __init__( self.optimizer = GradientOptimizer[opt_name](self.num_dims) self.strategy_name = "ARS" + # Set core kwargs es_params (lrate/sigma schedules) + self.lrate_init = lrate_init + self.lrate_decay = lrate_decay + self.lrate_limit = lrate_limit + self.sigma_init = sigma_init + self.sigma_decay = sigma_decay + self.sigma_limit = sigma_limit + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolution strategy.""" - return EvoParams(opt_params=self.optimizer.default_params) + opt_params = self.optimizer.default_params.replace( + lrate_init=self.lrate_init, + lrate_decay=self.lrate_decay, + lrate_limit=self.lrate_limit, + ) + return EvoParams( + opt_params=opt_params, + sigma_init=self.sigma_init, + sigma_decay=self.sigma_decay, + sigma_limit=self.sigma_limit, + ) def initialize_strategy( self, rng: chex.PRNGKey, params: EvoParams @@ -116,8 +140,5 @@ def tell_strategy( state.mean, theta_grad, state.opt_state, params.opt_params ) opt_state = self.optimizer.update(opt_state, params.opt_params) - - # Update lrate and standard deviation based on min and decay - sigma = state.sigma * params.sigma_decay - sigma = jnp.maximum(sigma, params.sigma_limit) + sigma = exp_decay(state.sigma, params.sigma_decay, params.sigma_limit) return state.replace(mean=mean, sigma=sigma, opt_state=opt_state) diff --git a/evosax/strategies/asebo.py b/evosax/strategies/asebo.py index 44fe70f..e404d88 100644 --- a/evosax/strategies/asebo.py +++ b/evosax/strategies/asebo.py @@ -3,7 +3,7 @@ import chex from typing import Tuple, Optional, Union from ..strategy import Strategy -from ..utils import GradientOptimizer, OptState, OptParams +from ..utils import GradientOptimizer, OptState, OptParams, exp_decay from flax import struct @@ -40,9 +40,15 @@ def __init__( popsize: int, num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, - subspace_dims: int = 2, + subspace_dims: int = 50, opt_name: str = "adam", - **fitness_kwargs: Union[bool, int, float] + lrate_init: float = 0.05, + lrate_decay: float = 1.0, + lrate_limit: float = 0.001, + sigma_init: float = 0.03, + sigma_decay: float = 1.0, + sigma_limit: float = 0.01, + **fitness_kwargs: Union[bool, int, float], ): """ASEBO (Choromanski et al., 2019) Reference: https://arxiv.org/abs/1903.04268 @@ -53,17 +59,37 @@ def __init__( super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) assert not self.popsize & 1, "Population size must be even" assert opt_name in ["sgd", "adam", "rmsprop", "clipup"] - assert ( - subspace_dims <= self.num_dims - ), "Subspace has to be smaller than optimization dims." self.optimizer = GradientOptimizer[opt_name](self.num_dims) - self.subspace_dims = subspace_dims + self.subspace_dims = min(subspace_dims, self.num_dims) + if self.subspace_dims < subspace_dims: + print( + "Subspace has to be smaller than optimization dims. Set to" + f" {self.subspace_dims} instead of {subspace_dims}." + ) self.strategy_name = "ASEBO" + # Set core kwargs es_params (lrate/sigma schedules) + self.lrate_init = lrate_init + self.lrate_decay = lrate_decay + self.lrate_limit = lrate_limit + self.sigma_init = sigma_init + self.sigma_decay = sigma_decay + self.sigma_limit = sigma_limit + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolution strategy.""" - return EvoParams(opt_params=self.optimizer.default_params) + opt_params = self.optimizer.default_params.replace( + lrate_init=self.lrate_init, + lrate_decay=self.lrate_decay, + lrate_limit=self.lrate_limit, + ) + return EvoParams( + opt_params=opt_params, + sigma_init=self.sigma_init, + sigma_decay=self.sigma_decay, + sigma_limit=self.sigma_limit, + ) def initialize_strategy( self, rng: chex.PRNGKey, params: EvoParams @@ -113,6 +139,12 @@ def svd_flip(u, v): U_ort = Vt[int(self.popsize / 2) :] UUT_ort = jnp.matmul(U_ort.T, U_ort) + + subspace_ready = state.gen_counter > self.subspace_dims + + UUT = jax.lax.select( + subspace_ready, UUT, jnp.zeros((self.num_dims, self.num_dims)) + ) cov = ( state.sigma * (state.alpha / self.num_dims) * jnp.eye(self.num_dims) + ((1 - state.alpha) / int(self.popsize / 2)) * UUT @@ -144,6 +176,8 @@ def tell_strategy( alpha = jnp.linalg.norm( jnp.dot(theta_grad, state.UUT_ort) ) / jnp.linalg.norm(jnp.dot(theta_grad, state.UUT)) + subspace_ready = state.gen_counter > self.subspace_dims + alpha = jax.lax.select(subspace_ready, alpha, 1.0) # Add grad FIFO-style to subspace archive (only if provided else FD) grad_subspace = jnp.zeros((self.subspace_dims, self.num_dims)) @@ -162,7 +196,7 @@ def tell_strategy( # Update lrate and standard deviation based on min and decay sigma = state.sigma * params.sigma_decay - sigma = jnp.maximum(sigma, params.sigma_limit) + sigma = exp_decay(state.sigma, params.sigma_decay, params.sigma_limit) return state.replace( mean=mean, sigma=sigma, opt_state=opt_state, alpha=alpha ) diff --git a/evosax/strategies/bipop_cma_es.py b/evosax/strategies/bipop_cma_es.py index 067d7a0..d9d5c17 100644 --- a/evosax/strategies/bipop_cma_es.py +++ b/evosax/strategies/bipop_cma_es.py @@ -25,6 +25,7 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, + sigma_init: float = 1.0, **fitness_kwargs: Union[bool, int, float] ): """BIPOP-CMA-ES (Hansen, 2009). @@ -37,7 +38,8 @@ def __init__( popsize=popsize, pholder_params=pholder_params, elite_ratio=elite_ratio, - **fitness_kwargs + sigma_init=sigma_init, + **fitness_kwargs, ) from ..restarts import BIPOP_Restarter from ..restarts.termination import spread_criterion, cma_criterion diff --git a/evosax/strategies/cma_es.py b/evosax/strategies/cma_es.py index 1b94ee6..84111e7 100755 --- a/evosax/strategies/cma_es.py +++ b/evosax/strategies/cma_es.py @@ -86,6 +86,7 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, + sigma_init: float = 1.0, **fitness_kwargs: Union[bool, int, float] ): """CMA-ES (e.g. Hansen, 2016) @@ -97,6 +98,9 @@ def __init__( self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) self.strategy_name = "CMA_ES" + # Set core kwargs es_params + self.sigma_init = sigma_init + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolution strategy.""" @@ -129,6 +133,7 @@ def params_strategy(self) -> EvoParams: d_sigma=d_sigma, c_c=c_c, chi_n=chi_n, + sigma_init=self.sigma_init, ) return params diff --git a/evosax/strategies/cr_fm_nes.py b/evosax/strategies/cr_fm_nes.py index 44a0634..70c23e3 100644 --- a/evosax/strategies/cr_fm_nes.py +++ b/evosax/strategies/cr_fm_nes.py @@ -79,6 +79,7 @@ def __init__( popsize: int, num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + sigma_init: float = 1.0, **fitness_kwargs: Union[bool, int, float] ): """Cost-Reduced Fast-Moving Natural ES (Nomura & Ono, 2022) @@ -88,6 +89,9 @@ def __init__( assert not self.popsize & 1, "Population size must be even" self.strategy_name = "CR_FM_NES" + # Set core kwargs es_params (sigma) + self.sigma_init = sigma_init + @property def default_params(self) -> EvoParams: """Return default parameters of evolutionary strategy.""" @@ -135,6 +139,7 @@ def default_params(self) -> EvoParams: chi_N=chi_N, alpha_dist=alpha_dist, h_inv=h_inv, + sigma_init=self.sigma_init, ) return params diff --git a/evosax/strategies/esmc.py b/evosax/strategies/esmc.py index 30fe820..058fafb 100644 --- a/evosax/strategies/esmc.py +++ b/evosax/strategies/esmc.py @@ -3,7 +3,7 @@ import chex from typing import Tuple, Optional, Union from ..strategy import Strategy -from ..utils import GradientOptimizer, OptState, OptParams +from ..utils import GradientOptimizer, OptState, OptParams, exp_decay from flax import struct @@ -38,6 +38,12 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, opt_name: str = "adam", + lrate_init: float = 0.05, + lrate_decay: float = 1.0, + lrate_limit: float = 0.001, + sigma_init: float = 0.03, + sigma_decay: float = 1.0, + sigma_limit: float = 0.01, **fitness_kwargs: Union[bool, int, float] ): """ESMC (Merchant et al., 2021) @@ -49,10 +55,28 @@ def __init__( self.optimizer = GradientOptimizer[opt_name](self.num_dims) self.strategy_name = "ESMC" + # Set core kwargs es_params (lrate/sigma schedules) + self.lrate_init = lrate_init + self.lrate_decay = lrate_decay + self.lrate_limit = lrate_limit + self.sigma_init = sigma_init + self.sigma_decay = sigma_decay + self.sigma_limit = sigma_limit + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolution strategy.""" - return EvoParams(opt_params=self.optimizer.default_params) + opt_params = self.optimizer.default_params.replace( + lrate_init=self.lrate_init, + lrate_decay=self.lrate_decay, + lrate_limit=self.lrate_limit, + ) + return EvoParams( + opt_params=opt_params, + sigma_init=self.sigma_init, + sigma_decay=self.sigma_decay, + sigma_limit=self.sigma_limit, + ) def initialize_strategy( self, rng: chex.PRNGKey, params: EvoParams diff --git a/evosax/strategies/full_iamalgam.py b/evosax/strategies/full_iamalgam.py index 23f7643..b959780 100644 --- a/evosax/strategies/full_iamalgam.py +++ b/evosax/strategies/full_iamalgam.py @@ -3,6 +3,7 @@ import chex from typing import Tuple, Optional, Union from ..strategy import Strategy +from ..utils import exp_decay from flax import struct @@ -29,7 +30,7 @@ class EvoParams: delta_ams: float = 2.0 theta_sdr: float = 1.0 c_mult_init: float = 1.0 - sigma_init: float = 0.0 + sigma_init: float = 0.1 sigma_decay: float = 0.999 sigma_limit: float = 0.0 init_min: float = 0.0 @@ -45,6 +46,9 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.35, + sigma_init: float = 0.0, + sigma_decay: float = 0.99, + sigma_limit: float = 0.0, **fitness_kwargs: Union[bool, int, float] ): """(Iterative) AMaLGaM (Bosman et al., 2013) - Full Covariance @@ -63,6 +67,11 @@ def __init__( self.ams_popsize = int(alpha_ams * (self.popsize - 1)) self.strategy_name = "Full_iAMaLGaM" + # Set core kwargs es_params + self.sigma_init = sigma_init + self.sigma_decay = sigma_decay + self.sigma_limit = sigma_limit + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolution strategy.""" @@ -79,8 +88,13 @@ def params_strategy(self) -> EvoParams: / (self.num_dims ** a_2_shift) ) - params = EvoParams(eta_sigma=eta_sigma, eta_shift=eta_shift) - return params + return EvoParams( + eta_sigma=eta_sigma, + eta_shift=eta_shift, + sigma_init=self.sigma_init, + sigma_decay=self.sigma_decay, + sigma_limit=self.sigma_limit, + ) def initialize_strategy( self, rng: chex.PRNGKey, params: EvoParams @@ -160,8 +174,7 @@ def tell_strategy( C = update_cov_amalgam(members_elite, state.C, mean, params.eta_sigma) # Decay isotropic part of Gaussian search distribution - sigma = state.sigma * params.sigma_decay - sigma = jnp.maximum(sigma, params.sigma_limit) + sigma = exp_decay(state.sigma, params.sigma_decay, params.sigma_limit) return state.replace( c_mult=c_mult, nis_counter=nis_counter, diff --git a/evosax/strategies/gld.py b/evosax/strategies/gld.py index 2957248..ae363f3 100644 --- a/evosax/strategies/gld.py +++ b/evosax/strategies/gld.py @@ -16,7 +16,7 @@ class EvoState: @struct.dataclass class EvoParams: - radius_max: float = 0.1 + radius_max: float = 0.2 radius_min: float = 0.001 radius_decay: float = 5 init_min: float = 0.0 diff --git a/evosax/strategies/guided_es.py b/evosax/strategies/guided_es.py index d5962bb..142517d 100644 --- a/evosax/strategies/guided_es.py +++ b/evosax/strategies/guided_es.py @@ -4,7 +4,7 @@ from typing import Tuple, Optional, Union from functools import partial from ..strategy import Strategy -from ..utils import GradientOptimizer, OptState, OptParams +from ..utils import GradientOptimizer, OptState, OptParams, exp_decay from flax import struct from evosax.utils import get_best_fitness_member @@ -40,9 +40,15 @@ def __init__( popsize: int, num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, - opt_name: str = "sgd", subspace_dims: int = 1, # k param in example notebook - **fitness_kwargs: Union[bool, int, float] + opt_name: str = "sgd", + lrate_init: float = 0.05, + lrate_decay: float = 1.0, + lrate_limit: float = 0.001, + sigma_init: float = 0.03, + sigma_decay: float = 1.0, + sigma_limit: float = 0.01, + **fitness_kwargs: Union[bool, int, float], ): """Guided ES (Maheswaranathan et al., 2018) Reference: https://arxiv.org/abs/1806.10230 @@ -55,9 +61,22 @@ def __init__( subspace_dims <= self.num_dims ), "Subspace has to be smaller than optimization dims." self.optimizer = GradientOptimizer[opt_name](self.num_dims) - self.subspace_dims = subspace_dims + self.subspace_dims = min(subspace_dims, self.num_dims) + if self.subspace_dims < subspace_dims: + print( + "Subspace has to be smaller than optimization dims. Set to" + f" {self.subspace_dims} instead of {subspace_dims}." + ) self.strategy_name = "GuidedES" + # Set core kwargs es_params (lrate/sigma schedules) + self.lrate_init = lrate_init + self.lrate_decay = lrate_decay + self.lrate_limit = lrate_limit + self.sigma_init = sigma_init + self.sigma_decay = sigma_decay + self.sigma_limit = sigma_limit + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolution strategy.""" @@ -155,8 +174,7 @@ def tell( opt_state = self.optimizer.update(opt_state, params.opt_params) # Update lrate and standard deviation based on min and decay - sigma = state.sigma * params.sigma_decay - sigma = jnp.maximum(sigma, params.sigma_limit) + sigma = exp_decay(state.sigma, params.sigma_decay, params.sigma_limit) state = state.replace(mean=mean, sigma=sigma, opt_state=opt_state) # Check if there is a new best member & update trackers diff --git a/evosax/strategies/indep_iamalgam.py b/evosax/strategies/indep_iamalgam.py index 21977b7..ef73628 100644 --- a/evosax/strategies/indep_iamalgam.py +++ b/evosax/strategies/indep_iamalgam.py @@ -3,6 +3,7 @@ import chex from typing import Tuple, Optional, Union from ..strategy import Strategy +from ..utils import exp_decay from .full_iamalgam import ( anticipated_mean_shift, adaptive_variance_scaling, @@ -50,6 +51,9 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.35, + sigma_init: float = 0.0, + sigma_decay: float = 0.99, + sigma_limit: float = 0.0, **fitness_kwargs: Union[bool, int, float] ): """(Iterative) AMaLGaM (Bosman et al., 2013) - Diagonal Covariance @@ -68,6 +72,11 @@ def __init__( self.ams_popsize = int(alpha_ams * (self.popsize - 1)) self.strategy_name = "Indep_iAMaLGaM" + # Set core kwargs es_params + self.sigma_init = sigma_init + self.sigma_decay = sigma_decay + self.sigma_limit = sigma_limit + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolution strategy.""" @@ -84,8 +93,13 @@ def params_strategy(self) -> EvoParams: / (self.num_dims ** a_2_shift) ) - params = EvoParams(eta_sigma=eta_sigma, eta_shift=eta_shift) - return params + return EvoParams( + eta_sigma=eta_sigma, + eta_shift=eta_shift, + sigma_init=self.sigma_init, + sigma_decay=self.sigma_decay, + sigma_limit=self.sigma_limit, + ) def initialize_strategy( self, rng: chex.PRNGKey, params: EvoParams @@ -165,8 +179,7 @@ def tell_strategy( C = update_cov_amalgam(members_elite, state.C, mean, params.eta_sigma) # Decay isotropic part of Gaussian search distribution - sigma = state.sigma * params.sigma_decay - sigma = jnp.maximum(sigma, params.sigma_limit) + sigma = exp_decay(state.sigma, params.sigma_decay, params.sigma_limit) return state.replace( c_mult=c_mult, nis_counter=nis_counter, diff --git a/evosax/strategies/ipop_cma_es.py b/evosax/strategies/ipop_cma_es.py index 8795df6..65d6fca 100644 --- a/evosax/strategies/ipop_cma_es.py +++ b/evosax/strategies/ipop_cma_es.py @@ -25,6 +25,7 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, + sigma_init: float = 1.0, **fitness_kwargs: Union[bool, int, float] ): """IPOP-CMA-ES (Auer & Hansen, 2005). @@ -37,6 +38,7 @@ def __init__( num_dims=num_dims, pholder_params=pholder_params, elite_ratio=elite_ratio, + sigma_init=sigma_init, **fitness_kwargs ) from ..restarts import IPOP_Restarter diff --git a/evosax/strategies/lm_ma_es.py b/evosax/strategies/lm_ma_es.py index 77d42c2..326b299 100644 --- a/evosax/strategies/lm_ma_es.py +++ b/evosax/strategies/lm_ma_es.py @@ -46,6 +46,7 @@ def __init__( pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, memory_size: int = 10, + sigma_init: float = 1.0, **fitness_kwargs: Union[bool, int, float] ): """Limited Memory MA-ES (Loshchilov et al., 2017) @@ -58,6 +59,9 @@ def __init__( self.memory_size = memory_size self.strategy_name = "LM_MA_ES" + # Set core kwargs es_params + self.sigma_init = sigma_init + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolution strategy.""" @@ -87,6 +91,7 @@ def params_strategy(self) -> EvoParams: d_sigma=d_sigma, chi_n=chi_n, mu_w=mu_w, + sigma_init=self.sigma_init, ) return params diff --git a/evosax/strategies/ma_es.py b/evosax/strategies/ma_es.py index b4611eb..86a7cfc 100644 --- a/evosax/strategies/ma_es.py +++ b/evosax/strategies/ma_es.py @@ -42,6 +42,7 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, + sigma_init: float = 1.0, **fitness_kwargs: Union[bool, int, float] ): """MA-ES (Bayer & Sendhoff, 2017) @@ -53,6 +54,9 @@ def __init__( self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) self.strategy_name = "MA_ES" + # Set core kwargs es_params + self.sigma_init = sigma_init + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolution strategy.""" @@ -82,6 +86,7 @@ def params_strategy(self) -> EvoParams: c_sigma=c_sigma, d_sigma=d_sigma, chi_n=chi_n, + sigma_init=self.sigma_init, ) return params diff --git a/evosax/strategies/open_es.py b/evosax/strategies/open_es.py index 8dff784..3a54af6 100755 --- a/evosax/strategies/open_es.py +++ b/evosax/strategies/open_es.py @@ -3,7 +3,7 @@ import chex from typing import Tuple, Optional, Union from ..strategy import Strategy -from ..utils import GradientOptimizer, OptState, OptParams +from ..utils import GradientOptimizer, OptState, OptParams, exp_decay from flax import struct @@ -36,6 +36,12 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, opt_name: str = "adam", + lrate_init: float = 0.05, + lrate_decay: float = 1.0, + lrate_limit: float = 0.001, + sigma_init: float = 0.03, + sigma_decay: float = 1.0, + sigma_limit: float = 0.01, **fitness_kwargs: Union[bool, int, float] ): """OpenAI-ES (Salimans et al. (2017) @@ -47,10 +53,28 @@ def __init__( self.optimizer = GradientOptimizer[opt_name](self.num_dims) self.strategy_name = "OpenES" + # Set core kwargs es_params (lrate/sigma schedules) + self.lrate_init = lrate_init + self.lrate_decay = lrate_decay + self.lrate_limit = lrate_limit + self.sigma_init = sigma_init + self.sigma_decay = sigma_decay + self.sigma_limit = sigma_limit + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolution strategy.""" - return EvoParams(opt_params=self.optimizer.default_params) + opt_params = self.optimizer.default_params.replace( + lrate_init=self.lrate_init, + lrate_decay=self.lrate_decay, + lrate_limit=self.lrate_limit, + ) + return EvoParams( + opt_params=opt_params, + sigma_init=self.sigma_init, + sigma_decay=self.sigma_decay, + sigma_limit=self.sigma_limit, + ) def initialize_strategy( self, rng: chex.PRNGKey, params: EvoParams @@ -102,6 +126,5 @@ def tell_strategy( state.mean, theta_grad, state.opt_state, params.opt_params ) opt_state = self.optimizer.update(opt_state, params.opt_params) - sigma = state.sigma * params.sigma_decay - sigma = jnp.maximum(sigma, params.sigma_limit) + sigma = exp_decay(state.sigma, params.sigma_decay, params.sigma_limit) return state.replace(mean=mean, sigma=sigma, opt_state=opt_state) diff --git a/evosax/strategies/persistent_es.py b/evosax/strategies/persistent_es.py index 118df7f..ee50fc1 100644 --- a/evosax/strategies/persistent_es.py +++ b/evosax/strategies/persistent_es.py @@ -3,7 +3,7 @@ import chex from typing import Tuple, Optional, Union from ..strategy import Strategy -from ..utils import GradientOptimizer, OptState, OptParams +from ..utils import GradientOptimizer, OptState, OptParams, exp_decay from flax import struct @@ -40,6 +40,12 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, opt_name: str = "adam", + lrate_init: float = 0.05, + lrate_decay: float = 1.0, + lrate_limit: float = 0.001, + sigma_init: float = 0.03, + sigma_decay: float = 1.0, + sigma_limit: float = 0.01, **fitness_kwargs: Union[bool, int, float] ): """Persistent ES (Vicol et al., 2021). @@ -52,10 +58,28 @@ def __init__( self.optimizer = GradientOptimizer[opt_name](self.num_dims) self.strategy_name = "PersistentES" + # Set core kwargs es_params (lrate/sigma schedules) + self.lrate_init = lrate_init + self.lrate_decay = lrate_decay + self.lrate_limit = lrate_limit + self.sigma_init = sigma_init + self.sigma_decay = sigma_decay + self.sigma_limit = sigma_limit + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolution strategy.""" - return EvoParams(opt_params=self.optimizer.default_params) + opt_params = self.optimizer.default_params.replace( + lrate_init=self.lrate_init, + lrate_decay=self.lrate_decay, + lrate_limit=self.lrate_limit, + ) + return EvoParams( + opt_params=opt_params, + sigma_init=self.sigma_init, + sigma_decay=self.sigma_decay, + sigma_limit=self.sigma_limit, + ) def initialize_strategy( self, rng: chex.PRNGKey, params: EvoParams @@ -112,8 +136,7 @@ def tell_strategy( opt_state = self.optimizer.update(opt_state, params.opt_params) inner_step_counter = state.inner_step_counter + params.K - sigma = state.sigma * params.sigma_decay - sigma = jnp.maximum(sigma, params.sigma_limit) + sigma = exp_decay(state.sigma, params.sigma_decay, params.sigma_limit) # Reset accumulated antithetic noise memory if done with inner problem reset = inner_step_counter >= params.T inner_step_counter = jax.lax.select(reset, 0, inner_step_counter) diff --git a/evosax/strategies/pgpe.py b/evosax/strategies/pgpe.py index bc1ea4e..6057fb7 100755 --- a/evosax/strategies/pgpe.py +++ b/evosax/strategies/pgpe.py @@ -3,7 +3,7 @@ import chex from typing import Tuple, Optional, Union from ..strategy import Strategy -from ..utils import GradientOptimizer, OptState, OptParams +from ..utils import GradientOptimizer, OptState, OptParams, exp_decay from flax import struct @@ -39,6 +39,12 @@ def __init__( pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 1.0, opt_name: str = "adam", + lrate_init: float = 0.05, + lrate_decay: float = 1.0, + lrate_limit: float = 0.001, + sigma_init: float = 0.03, + sigma_decay: float = 1.0, + sigma_limit: float = 0.01, **fitness_kwargs: Union[bool, int, float] ): """PGPE (e.g. Sehnke et al., 2010) @@ -54,10 +60,28 @@ def __init__( self.optimizer = GradientOptimizer[opt_name](self.num_dims) self.strategy_name = "PGPE" + # Set core kwargs es_params (lrate/sigma schedules) + self.lrate_init = lrate_init + self.lrate_decay = lrate_decay + self.lrate_limit = lrate_limit + self.sigma_init = sigma_init + self.sigma_decay = sigma_decay + self.sigma_limit = sigma_limit + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolution strategy.""" - return EvoParams(opt_params=self.optimizer.default_params) + opt_params = self.optimizer.default_params.replace( + lrate_init=self.lrate_init, + lrate_decay=self.lrate_decay, + lrate_limit=self.lrate_limit, + ) + return EvoParams( + opt_params=opt_params, + sigma_init=self.sigma_init, + sigma_decay=self.sigma_decay, + sigma_limit=self.sigma_limit, + ) def initialize_strategy( self, rng: chex.PRNGKey, params: EvoParams @@ -135,6 +159,5 @@ def tell_strategy( min_allowed, max_allowed, ) - sigma = sigma * params.sigma_decay - sigma = jnp.maximum(sigma, params.sigma_limit) + sigma = exp_decay(state.sigma, params.sigma_decay, params.sigma_limit) return state.replace(mean=mean, sigma=sigma, opt_state=opt_state) diff --git a/evosax/strategies/rm_es.py b/evosax/strategies/rm_es.py index 59f5791..105bc39 100644 --- a/evosax/strategies/rm_es.py +++ b/evosax/strategies/rm_es.py @@ -67,6 +67,7 @@ def __init__( pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, memory_size: int = 10, + sigma_init: float = 1.0, **fitness_kwargs: Union[bool, int, float] ): """Rank-m ES (Li & Zhang, 2017) @@ -79,6 +80,9 @@ def __init__( self.memory_size = memory_size # number of ranks self.strategy_name = "RmES" + # Set core kwargs es_params + self.sigma_init = sigma_init + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolution strategy.""" @@ -91,6 +95,7 @@ def params_strategy(self) -> EvoParams: c_c=c_c, c_sigma=jnp.minimum(2 / (self.num_dims + 7), 0.05), mu_eff=mu_eff, + sigma_init=self.sigma_init, ) return params diff --git a/evosax/strategies/sep_cma_es.py b/evosax/strategies/sep_cma_es.py index d442680..a80f2ab 100644 --- a/evosax/strategies/sep_cma_es.py +++ b/evosax/strategies/sep_cma_es.py @@ -63,6 +63,7 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, + sigma_init: float = 1.0, **fitness_kwargs: Union[bool, int, float] ): """Separable CMA-ES (e.g. Ros & Hansen, 2008) @@ -75,6 +76,9 @@ def __init__( self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) self.strategy_name = "Sep_CMA_ES" + # Set core kwargs es_params + self.sigma_init = sigma_init + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolution strategy.""" @@ -114,6 +118,7 @@ def params_strategy(self) -> EvoParams: d_sigma=d_sigma, c_c=c_c, chi_n=chi_n, + sigma_init=self.sigma_init, ) return params diff --git a/evosax/strategies/sim_anneal.py b/evosax/strategies/sim_anneal.py index a805b2b..5b73bec 100644 --- a/evosax/strategies/sim_anneal.py +++ b/evosax/strategies/sim_anneal.py @@ -38,6 +38,9 @@ def __init__( popsize: int, num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + sigma_init: float = 0.03, + sigma_decay: float = 1.0, + sigma_limit: float = 0.01, **fitness_kwargs: Union[bool, int, float] ): """Simulated Annealing (Rasdi Rere et al., 2015) @@ -46,10 +49,19 @@ def __init__( super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) self.strategy_name = "SimAnneal" + # Set core kwargs es_params (lrate/sigma schedules) + self.sigma_init = sigma_init + self.sigma_decay = sigma_decay + self.sigma_limit = sigma_limit + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolution strategy.""" - return EvoParams() + return EvoParams( + sigma_init=self.sigma_init, + sigma_decay=self.sigma_decay, + sigma_limit=self.sigma_limit, + ) def initialize_strategy( self, rng: chex.PRNGKey, params: EvoParams diff --git a/evosax/strategies/simple_es.py b/evosax/strategies/simple_es.py index 27ed031..ac1d57d 100755 --- a/evosax/strategies/simple_es.py +++ b/evosax/strategies/simple_es.py @@ -34,6 +34,7 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, + sigma_init: float = 1.0, **fitness_kwargs: Union[bool, int, float] ): """Simple Gaussian Evolution Strategy (Rechenberg, 1975) @@ -44,11 +45,14 @@ def __init__( self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) self.strategy_name = "SimpleES" + # Set core kwargs es_params + self.sigma_init = sigma_init + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolution strategy.""" # Only parents have positive weight - equal weighting! - return EvoParams() + return EvoParams(sigma_init=self.sigma_init) def initialize_strategy( self, rng: chex.PRNGKey, params: EvoParams diff --git a/evosax/strategies/simple_ga.py b/evosax/strategies/simple_ga.py index bfdbbb0..d29832a 100755 --- a/evosax/strategies/simple_ga.py +++ b/evosax/strategies/simple_ga.py @@ -3,6 +3,7 @@ import chex from typing import Tuple, Optional, Union from ..strategy import Strategy +from ..utils import exp_decay from flax import struct @@ -19,7 +20,7 @@ class EvoState: @struct.dataclass class EvoParams: - cross_over_rate: float = 0.5 + cross_over_rate: float = 0.1 sigma_init: float = 0.07 sigma_decay: float = 0.999 sigma_limit: float = 0.01 @@ -36,6 +37,9 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, elite_ratio: float = 0.5, + sigma_init: float = 0.1, + sigma_decay: float = 1.0, + sigma_limit: float = 0.01, **fitness_kwargs: Union[bool, int, float] ): """Simple Genetic Algorithm (Such et al., 2017) @@ -47,10 +51,19 @@ def __init__( self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) self.strategy_name = "SimpleGA" + # Set core kwargs es_params + self.sigma_init = sigma_init + self.sigma_decay = sigma_decay + self.sigma_limit = sigma_limit + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolution strategy.""" - return EvoParams() + return EvoParams( + sigma_init=self.sigma_init, + sigma_decay=self.sigma_decay, + sigma_limit=self.sigma_limit, + ) def initialize_strategy( self, rng: chex.PRNGKey, params: EvoParams @@ -118,11 +131,7 @@ def tell_strategy( fitness = fitness[idx] archive = solution[idx] # Update mutation epsilon - multiplicative decay - sigma = jax.lax.select( - state.sigma > params.sigma_limit, - state.sigma * params.sigma_decay, - state.sigma, - ) + sigma = exp_decay(state.sigma, params.sigma_decay, params.sigma_limit) # Keep mean across stored archive around for evaluation protocol mean = archive[0] return state.replace( diff --git a/evosax/strategies/snes.py b/evosax/strategies/snes.py index ce8a82e..e80e02d 100644 --- a/evosax/strategies/snes.py +++ b/evosax/strategies/snes.py @@ -44,6 +44,7 @@ def __init__( popsize: int, num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + sigma_init: float = 1.0, **fitness_kwargs: Union[bool, int, float] ): """Separable Exponential Natural ES (Wierstra et al., 2014) @@ -52,13 +53,16 @@ def __init__( super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) self.strategy_name = "SNES" + # Set core kwargs es_params + self.sigma_init = sigma_init + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolutionary strategy.""" lrate_sigma = (3 + jnp.log(self.num_dims)) / ( 5 * jnp.sqrt(self.num_dims) ) - params = EvoParams(lrate_sigma=lrate_sigma) + params = EvoParams(lrate_sigma=lrate_sigma, sigma_init=self.sigma_init) return params def initialize_strategy( diff --git a/evosax/strategies/xnes.py b/evosax/strategies/xnes.py index d4d31f6..609f8f5 100644 --- a/evosax/strategies/xnes.py +++ b/evosax/strategies/xnes.py @@ -41,6 +41,7 @@ def __init__( popsize: int, num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + sigma_init: float = 1.0, **fitness_kwargs: Union[bool, int, float] ): """Exponential Natural ES (Wierstra et al., 2014) @@ -49,6 +50,9 @@ def __init__( super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) self.strategy_name = "xNES" + # Set core kwargs es_params + self.sigma_init = sigma_init + @property def params_strategy(self) -> EvoParams: """Return default parameters of evolutionary strategy.""" @@ -57,7 +61,10 @@ def params_strategy(self) -> EvoParams: ) rho = 0.5 - 1.0 / (3 * (self.num_dims + 1)) params = EvoParams( - lrate_sigma_init=lrate_sigma, lrate_B=lrate_sigma, rho=rho + lrate_sigma_init=lrate_sigma, + lrate_B=lrate_sigma, + rho=rho, + sigma_init=self.sigma_init, ) return params diff --git a/evosax/strategy.py b/evosax/strategy.py index 834c705..469110a 100755 --- a/evosax/strategy.py +++ b/evosax/strategy.py @@ -147,3 +147,11 @@ def tell_strategy( ) -> EvoState: """Search-specific `tell` update. Returns updated state.""" raise NotImplementedError + + def get_eval_params(self, state: EvoState): + """Return reshaped parameters to evaluate.""" + if self.use_param_reshaper: + x_out = self.param_reshaper.reshape_single(state.mean) + else: + x_out = state.mean + return x_out diff --git a/evosax/utils/__init__.py b/evosax/utils/__init__.py index c93ee75..a8b4203 100755 --- a/evosax/utils/__init__.py +++ b/evosax/utils/__init__.py @@ -11,7 +11,16 @@ from .helpers import get_best_fitness_member # Import Gradient Based Optimizer step functions -from .optimizer import SGD, Adam, RMSProp, ClipUp, Adan, OptState, OptParams +from .optimizer import ( + SGD, + Adam, + RMSProp, + ClipUp, + Adan, + OptState, + OptParams, + exp_decay, +) GradientOptimizer = { "sgd": SGD, @@ -35,4 +44,5 @@ "Adan", "OptState", "OptParams", + "exp_decay", ] diff --git a/evosax/utils/helpers.py b/evosax/utils/helpers.py index 02c3890..87e93c2 100644 --- a/evosax/utils/helpers.py +++ b/evosax/utils/helpers.py @@ -7,6 +7,7 @@ def get_best_fitness_member( x: chex.Array, fitness: chex.Array, state ) -> Tuple[chex.Array, float]: + """Check if fitness improved & replace in ES state.""" best_in_gen = jnp.argmin(fitness) best_in_gen_fitness, best_in_gen_member = ( fitness[best_in_gen], diff --git a/evosax/utils/optimizer.py b/evosax/utils/optimizer.py index fd25d32..e686c96 100644 --- a/evosax/utils/optimizer.py +++ b/evosax/utils/optimizer.py @@ -11,6 +11,15 @@ # "clip_value": 5, +def exp_decay( + param: chex.Array, param_decay: chex.Array, param_limit: chex.Array +) -> chex.Array: + """Exponentially decay parameter & clip by minimal value.""" + param = param * param_decay + param = jnp.maximum(param, param_limit) + return param + + @struct.dataclass class OptState: lrate: float @@ -60,8 +69,7 @@ def step( def update(self, state: OptState, params: OptParams) -> OptState: """Exponentially decay the learning rate if desired.""" - lrate = state.lrate * params.lrate_decay - lrate = jnp.maximum(lrate, params.lrate_limit) + lrate = exp_decay(state.lrate, params.lrate_decay, params.lrate_limit) return state.replace(lrate=lrate) @property @@ -94,7 +102,7 @@ def __init__(self, num_dims: int): def params_opt(self) -> Dict[str, float]: """Return default SGD+Momentum parameters.""" return { - "momentum": 0.9, + "momentum": 0.0, } def initialize_opt(self, params: OptParams) -> OptState: diff --git a/evosax/utils/reshape_fitness.py b/evosax/utils/reshape_fitness.py index 1f1d7a7..58c803d 100755 --- a/evosax/utils/reshape_fitness.py +++ b/evosax/utils/reshape_fitness.py @@ -30,14 +30,18 @@ def __init__( @partial(jax.jit, static_argnums=(0,)) def apply(self, x: chex.Array, fitness: chex.Array) -> chex.Array: """Max objective trafo, rank shaping, z scoring & add weight decay.""" - fitness = jax.lax.select(self.maximize, -1 * fitness, fitness) - fitness = jax.lax.select( - self.centered_rank, centered_rank_trafo(fitness), fitness - ) - fitness = jax.lax.select(self.z_score, z_score_trafo(fitness), fitness) - fitness = jax.lax.select( - self.norm_range, range_norm_trafo(fitness, -1.0, 1.0), fitness - ) + if self.maximize: + fitness = -1 * fitness + + if self.centered_rank: + fitness = centered_rank_trafo(fitness) + + if self.z_score: + fitness = z_score_trafo(fitness) + + if self.norm_range: + fitness = range_norm_trafo(fitness, -1.0, 1.0) + # "Reduce" fitness based on L2 norm of parameters if self.w_decay > 0.0: l2_fit_red = self.w_decay * compute_l2_norm(x) diff --git a/evosax/utils/reshape_params.py b/evosax/utils/reshape_params.py index c3fb5fe..bab0dfc 100755 --- a/evosax/utils/reshape_params.py +++ b/evosax/utils/reshape_params.py @@ -57,7 +57,7 @@ def __init__( def reshape(self, x: chex.Array) -> chex.ArrayTree: """Perform reshaping for a 2D matrix (pop_members, params).""" - vmap_shape = jax.vmap(self.reshape_single, in_axes=(0,)) + vmap_shape = jax.vmap(self.reshape_single) if self.n_devices > 1: x = self.split_params_for_pmap(x) map_shape = jax.pmap(vmap_shape) @@ -65,6 +65,12 @@ def reshape(self, x: chex.Array) -> chex.ArrayTree: map_shape = vmap_shape return map_shape(x) + def multi_reshape(self, x: chex.Array) -> chex.ArrayTree: + """Reshape parameters lying already on different devices.""" + # No reshaping required! + vmap_shape = jax.vmap(self.reshape_single) + return jax.pmap(vmap_shape)(x) + def flatten(self, x: chex.ArrayTree) -> chex.Array: """Reshaping pytree parameters into flat array.""" vmap_flat = jax.vmap(ravel_pytree) @@ -79,6 +85,12 @@ def map_flat(x): flat = map_flat(x) return flat + def multi_flatten(self, x: chex.Array) -> chex.ArrayTree: + """Flatten parameters lying remaining on different devices.""" + # No reshaping required! + vmap_flat = jax.vmap(ravel_pytree) + return jax.pmap(vmap_flat)(x) + def split_params_for_pmap(self, param: chex.Array) -> chex.Array: """Helper reshapes param (bs, #params) into (#dev, bs/#dev, #params).""" return jnp.stack(jnp.split(param, self.n_devices)) From b0fe38fef9693b2b7d36b97ebfee01ae9d96b54b Mon Sep 17 00:00:00 2001 From: RobertTLange Date: Sun, 27 Nov 2022 12:16:04 +0100 Subject: [PATCH 10/13] Fix GESMR-GA & add MR15-GA --- CHANGELOG.md | 2 + README.md | 1 + evosax/__init__.py | 3 + evosax/strategies/__init__.py | 2 + evosax/strategies/gesmr_ga.py | 8 +- evosax/strategies/mr15_ga.py | 139 +++++++++++++++++++++++++++++++++ evosax/strategies/samr_ga.py | 6 +- evosax/strategies/simple_ga.py | 11 +-- evosax/utils/evojax_wrapper.py | 5 +- tests/conftest.py | 2 + 10 files changed, 167 insertions(+), 12 deletions(-) create mode 100644 evosax/strategies/mr15_ga.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 119099a..3682c90 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - Guided ES (Maheswaranathan et al., 2018) - ASEBO (Choromanski et al., 2019) - CR-FM-NES (Nomura & Ono, 2022) + - MR15-GA (Rechenberg, 1978) - Adds full set of BBOB low-dimensional functions (`BBOBFitness`) - Adds 2D visualizer animating sampled points (`BBOBVisualizer`) - Adds `Evosax2JAXWrapper` to wrap all evosax strategies @@ -34,6 +35,7 @@ - `ParameterReshaper` can now be directly applied from within the strategy. You simply have to provide a `pholder_params` pytree at strategy instantiation (and no `num_dims`). - `FitnessShaper` can also be directly applied from within the strategy. This makes it easier to track the best performing member across generations and addresses issue #32. Simply provide the fitness shaping settings as args to the strategy (`maximize`, `centered_rank`, ...) - Removes Brax fitness (use EvoJAX version instead) +- Add lrate and sigma schedule to strategy instantiation ##### Fixed diff --git a/README.md b/README.md index d8dc3c2..f4b0c07 100755 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ state.best_member, state.best_fitness | Guided ES | [Maheswaranathan et al. (2018)](https://arxiv.org/abs/1806.10230) | [`GuidedES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/guided_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | ASEBO | [Choromanski et al. (2019)](https://arxiv.org/abs/1903.04268) | [`GuidedES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/asebo.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | CR-FM-NES | [Nomura & Ono (2022)](https://arxiv.org/abs/2201.11422) | [`CR-FM-NES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/cr_fm_nes.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| MR15-GA | [Rechenberg (1978)](https://link.springer.com/chapter/10.1007/978-3-642-81283-5_8) | [`MR15_GA`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/mr15_ga.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) ## Installation ⏳ diff --git a/evosax/__init__.py b/evosax/__init__.py index 59ca684..9bc366b 100755 --- a/evosax/__init__.py +++ b/evosax/__init__.py @@ -29,6 +29,7 @@ GuidedES, ASEBO, CR_FM_NES, + MR15_GA, ) from .utils import FitnessShaper, ParameterReshaper, ESLog from .networks import NetworkMapper @@ -65,6 +66,7 @@ "GuidedES": GuidedES, "ASEBO": ASEBO, "CR_FM_NES": CR_FM_NES, + "MR15_GA": MR15_GA, } __all__ = [ @@ -106,4 +108,5 @@ "GuidedES", "ASEBO", "CR_FM_NES", + "MR15_GA", ] diff --git a/evosax/strategies/__init__.py b/evosax/strategies/__init__.py index 15bcda3..b28992e 100755 --- a/evosax/strategies/__init__.py +++ b/evosax/strategies/__init__.py @@ -27,6 +27,7 @@ from .guided_es import GuidedES from .asebo import ASEBO from .cr_fm_nes import CR_FM_NES +from .mr15_ga import MR15_GA __all__ = [ @@ -59,4 +60,5 @@ "GuidedES", "ASEBO", "CR_FM_NES", + "MR15_GA", ] diff --git a/evosax/strategies/gesmr_ga.py b/evosax/strategies/gesmr_ga.py index acc256d..1a67ee0 100644 --- a/evosax/strategies/gesmr_ga.py +++ b/evosax/strategies/gesmr_ga.py @@ -73,7 +73,7 @@ def initialize_strategy( rng=rng, mean=initialization[0], archive=initialization, - fitness=jnp.zeros(self.popsize) + jnp.finfo(jnp.float32).max, + fitness=jnp.zeros(self.elite_popsize) + jnp.finfo(jnp.float32).max, sigma=jnp.zeros(self.num_sigma_groups) + params.sigma_init, best_member=initialization[0], ) @@ -153,10 +153,14 @@ def tell_strategy( (self.num_sigma_groups - 1,), ) sigma = jnp.concatenate([state.sigma[0][None], sigma_elite[idx_s]]) + + # Set mean to best member seen so far + improved = fitness[0] < state.best_fitness + best_mean = jax.lax.select(improved, archive[0], state.best_member) return state.replace( rng=rng, fitness=fitness[idx], archive=archive, sigma=sigma, - mean=archive[0], + mean=best_mean, ) diff --git a/evosax/strategies/mr15_ga.py b/evosax/strategies/mr15_ga.py new file mode 100644 index 0000000..067731f --- /dev/null +++ b/evosax/strategies/mr15_ga.py @@ -0,0 +1,139 @@ +import jax +import jax.numpy as jnp +import chex +from typing import Tuple, Optional, Union +from ..strategy import Strategy +from .simple_ga import single_mate +from flax import struct + + +@struct.dataclass +class EvoState: + mean: chex.Array + archive: chex.Array + fitness: chex.Array + sigma: chex.Array + best_member: chex.Array + best_fitness: float = jnp.finfo(jnp.float32).max + gen_counter: int = 0 + + +@struct.dataclass +class EvoParams: + cross_over_rate: float = 0.0 + sigma_init: float = 0.07 + sigma_ratio: float = 0.15 + init_min: float = 0.0 + init_max: float = 0.0 + clip_min: float = -jnp.finfo(jnp.float32).max + clip_max: float = jnp.finfo(jnp.float32).max + + +class MR15_GA(Strategy): + def __init__( + self, + popsize: int, + num_dims: Optional[int] = None, + pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, + elite_ratio: float = 0.0, + sigma_ratio: float = 0.15, + sigma_init: float = 0.1, + **fitness_kwargs: Union[bool, int, float] + ): + """1/5 MR Genetic Algorithm (Rechenberg, 1987) + Reference: https://link.springer.com/chapter/10.1007/978-3-642-81283-5_8 + """ + + super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) + self.elite_ratio = elite_ratio + self.elite_popsize = max(1, int(self.popsize * self.elite_ratio)) + self.strategy_name = "MR15_GA" + + # Set core kwargs es_params + self.sigma_ratio = sigma_ratio # no. mutation that have to improve + self.sigma_init = sigma_init + + @property + def params_strategy(self) -> EvoParams: + """Return default parameters of evolution strategy.""" + return EvoParams( + sigma_init=self.sigma_init, sigma_ratio=self.sigma_ratio + ) + + def initialize_strategy( + self, rng: chex.PRNGKey, params: EvoParams + ) -> EvoState: + """`initialize` the differential evolution strategy.""" + initialization = jax.random.uniform( + rng, + (self.elite_popsize, self.num_dims), + minval=params.init_min, + maxval=params.init_max, + ) + state = EvoState( + mean=initialization.mean(axis=0), + archive=initialization, + fitness=jnp.zeros(self.elite_popsize) + jnp.finfo(jnp.float32).max, + sigma=params.sigma_init, + best_member=initialization.mean(axis=0), + ) + return state + + def ask_strategy( + self, rng: chex.PRNGKey, state: EvoState, params: EvoParams + ) -> Tuple[chex.Array, EvoState]: + """ + `ask` for new proposed candidates to evaluate next. + 1. For each member of elite: + - Sample two current elite members (a & b) + - Cross over all dims of a with corresponding one from b + if random number > co-rate + - Additionally add noise on top of all elite parameters + """ + rng, rng_eps, rng_idx_a, rng_idx_b = jax.random.split(rng, 4) + rng_mate = jax.random.split(rng, self.popsize) + epsilon = ( + jax.random.normal(rng_eps, (self.popsize, self.num_dims)) + * state.sigma + ) + elite_ids = jnp.arange(self.elite_popsize) + idx_a = jax.random.choice(rng_idx_a, elite_ids, (self.popsize,)) + idx_b = jax.random.choice(rng_idx_b, elite_ids, (self.popsize,)) + members_a = state.archive[idx_a] + members_b = state.archive[idx_b] + x = jax.vmap(single_mate, in_axes=(0, 0, 0, None))( + rng_mate, members_a, members_b, params.cross_over_rate + ) + x += epsilon + return jnp.squeeze(x), state + + def tell_strategy( + self, + x: chex.Array, + fitness: chex.Array, + state: EvoState, + params: EvoParams, + ) -> EvoState: + """ + `tell` update to ES state. + If fitness of y <= fitness of x -> replace in population. + """ + # Combine current elite and recent generation info + fitness = jnp.concatenate([fitness, state.fitness]) + solution = jnp.concatenate([x, state.archive]) + # Select top elite from total archive info + idx = jnp.argsort(fitness)[0 : self.elite_popsize] + fitness = fitness[idx] + archive = solution[idx] + # Update mutation sigma - double if more than 15% improved + good_mutations_ratio = jnp.mean(fitness < state.best_fitness) + increase_sigma = good_mutations_ratio > params.sigma_ratio + sigma = jax.lax.select( + increase_sigma, 2 * state.sigma, 0.5 * state.sigma + ) + # Set mean to best member seen so far + improved = fitness[0] < state.best_fitness + best_mean = jax.lax.select(improved, archive[0], state.best_member) + return state.replace( + fitness=fitness, archive=archive, sigma=sigma, mean=best_mean + ) diff --git a/evosax/strategies/samr_ga.py b/evosax/strategies/samr_ga.py index d4a8a49..c287ac1 100644 --- a/evosax/strategies/samr_ga.py +++ b/evosax/strategies/samr_ga.py @@ -101,6 +101,10 @@ def tell_strategy( fitness = fitness[idx] archive = x[idx] sigma = state.sigma[idx] + + # Set mean to best member seen so far + improved = fitness[0] < state.best_fitness + best_mean = jax.lax.select(improved, archive[0], state.best_member) return state.replace( - fitness=fitness, archive=archive, sigma=sigma, mean=archive[0] + fitness=fitness, archive=archive, sigma=sigma, mean=best_mean ) diff --git a/evosax/strategies/simple_ga.py b/evosax/strategies/simple_ga.py index d29832a..17ec9a5 100755 --- a/evosax/strategies/simple_ga.py +++ b/evosax/strategies/simple_ga.py @@ -20,9 +20,9 @@ class EvoState: @struct.dataclass class EvoParams: - cross_over_rate: float = 0.1 + cross_over_rate: float = 0.0 sigma_init: float = 0.07 - sigma_decay: float = 0.999 + sigma_decay: float = 1.0 sigma_limit: float = 0.01 init_min: float = 0.0 init_max: float = 0.0 @@ -132,10 +132,11 @@ def tell_strategy( archive = solution[idx] # Update mutation epsilon - multiplicative decay sigma = exp_decay(state.sigma, params.sigma_decay, params.sigma_limit) - # Keep mean across stored archive around for evaluation protocol - mean = archive[0] + # Set mean to best member seen so far + improved = fitness[0] < state.best_fitness + best_mean = jax.lax.select(improved, archive[0], state.best_member) return state.replace( - fitness=fitness, archive=archive, sigma=sigma, mean=mean + fitness=fitness, archive=archive, sigma=sigma, mean=best_mean ) diff --git a/evosax/utils/evojax_wrapper.py b/evosax/utils/evojax_wrapper.py index 7f36efb..1c941cb 100644 --- a/evosax/utils/evojax_wrapper.py +++ b/evosax/utils/evojax_wrapper.py @@ -19,12 +19,9 @@ def __init__( seed: int = 42, ): self.es = evosax_strategy( - popsize=pop_size, num_dims=param_size, **es_config + popsize=pop_size, num_dims=param_size, **es_config, **opt_params ) self.es_params = self.es.default_params.replace(**es_params) - if len(opt_params.keys()) > 0: - opt_params = self.es_params.opt_params.replace(**opt_params) - self.es_params = self.es_params.replace(opt_params=opt_params) self.pop_size = pop_size self.param_size = param_size self.rand_key = jax.random.PRNGKey(seed=seed) diff --git a/tests/conftest.py b/tests/conftest.py index 49a755f..e251f8e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -33,6 +33,8 @@ def pytest_generate_tests(metafunc): # "GESMR_GA", "GuidedES", "ASEBO", + "CR_FM_NES", + "MR15_GA", ], ) else: From 25fd217e533fdf85c795b7257eef0d73b8733b4c Mon Sep 17 00:00:00 2001 From: RobertTLange Date: Wed, 30 Nov 2022 19:33:09 +0100 Subject: [PATCH 11/13] Update notebooks --- .gitignore | 1 + README.md | 35 +- evosax/experimental/decodings/hyper.py | 16 +- .../decodings}/hyper_networks.py | 0 evosax/experimental/decodings/random.py | 4 + evosax/networks/__init__.py | 2 - evosax/problems/__init__.py | 6 +- evosax/problems/control_gym.py | 8 +- evosax/problems/sequence.py | 4 +- evosax/problems/vision.py | 4 +- evosax/strategies/de.py | 2 +- evosax/strategies/snes.py | 28 +- evosax/strategy.py | 4 +- evosax/utils/__init__.py | 3 +- evosax/utils/helpers.py | 16 +- evosax/utils/visualizer_2d.py | 73 +++- examples/00_getting_started.ipynb | 350 ++++++++++++++--- examples/01_classic_benchmark.ipynb | 259 ++++++------ examples/02_mlp_control.ipynb | 112 ++---- examples/03_cnn_mnist.ipynb | 37 +- examples/04_lrate_pes.ipynb | 60 ++- examples/05_quadratic_pbt.ipynb | 22 +- examples/06_restart_es.ipynb | 163 ++++---- examples/07_brax_control.ipynb | 188 ++++----- examples/08_encodings.ipynb | 276 ------------- examples/09_exp_batch_es.ipynb | 367 ------------------ tests/test_fitness_rollout.py | 20 +- 27 files changed, 796 insertions(+), 1264 deletions(-) rename evosax/{networks => experimental/decodings}/hyper_networks.py (100%) delete mode 100644 examples/08_encodings.ipynb delete mode 100644 examples/09_exp_batch_es.ipynb diff --git a/.gitignore b/.gitignore index 225d5f0..9d540ed 100755 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +examples/experimental bbob.py # Standard ROB excludes .sync-config.cson diff --git a/README.md b/README.md index f4b0c07..67477a1 100755 --- a/README.md +++ b/README.md @@ -36,15 +36,14 @@ state.best_member, state.best_fitness | OpenES | [Salimans et al. (2017)](https://arxiv.org/pdf/1703.03864.pdf) | [`OpenES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/open_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/03_cnn_mnist.ipynb) | PGPE | [Sehnke et al. (2010)](https://citeseerx.ist.psu.edu/viewdoc/download;jsessionid=A64D1AE8313A364B814998E9E245B40A?doi=10.1.1.180.7104&rep=rep1&type=pdf) | [`PGPE`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/pgpe.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/02_mlp_control.ipynb) | ARS | [Mania et al. (2018)](https://arxiv.org/pdf/1803.07055.pdf) | [`ARS`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/ars.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/00_getting_started.ipynb) -| CMA-ES | [Hansen & Ostermeier (2001)](http://www.cmap.polytechnique.fr/~nikolaus.hansen/cmaartic.pdf) | [`CMA_ES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/cma_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) -| Simple Gaussian | [Rechenberg (1978)](https://link.springer.com/chapter/10.1007/978-3-642-81283-5_8) | [`SimpleES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/simple_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) -| Simple Genetic | [Such et al. (2017)](https://arxiv.org/abs/1712.06567) | [`SimpleGA`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/simple_ga.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) -| XNES | [Wierstra et al. (2014)](https://www.jmlr.org/papers/volume15/wierstra14a/wierstra14a.pdf) | [`XNES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/xnes.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) -| SNES | [Wierstra et al. (2014)](https://www.jmlr.org/papers/volume15/wierstra14a/wierstra14a.pdf) | [`SNES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/sxnes.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) -| Particle Swarm Optimization | [Kennedy & Eberhart (1995)](https://ieeexplore.ieee.org/document/488968) | [`PSO`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/pso.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) -| Differential Evolution | [Storn & Price (1997)](https://www.metabolic-economics.de/pages/seminar_theoretische_biologie_2007/literatur/schaber/Storn1997JGlobOpt11.pdf) | [`DE`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/de.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| ESMC | [Merchant et al. (2021)](https://proceedings.mlr.press/v139/merchant21a.html) | [`ESMC`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/esmc.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | Persistent ES | [Vicol et al. (2021)](http://proceedings.mlr.press/v139/vicol21a.html) | [`PersistentES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/persistent_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/04_lrate_pes.ipynb) -| Population-Based Training | [Jaderberg et al. (2017)](https://arxiv.org/abs/1711.09846) | [`PBT`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/pbt.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/05_quadratic_pbt.ipynb) +| xNES | [Wierstra et al. (2014)](https://www.jmlr.org/papers/volume15/wierstra14a/wierstra14a.pdf) | [`XNES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/xnes.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| SNES | [Wierstra et al. (2014)](https://www.jmlr.org/papers/volume15/wierstra14a/wierstra14a.pdf) | [`SNES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/sxnes.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| CR-FM-NES | [Nomura & Ono (2022)](https://arxiv.org/abs/2201.11422) | [`CR-FM-NES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/cr_fm_nes.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| Guided ES | [Maheswaranathan et al. (2018)](https://arxiv.org/abs/1806.10230) | [`GuidedES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/guided_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| ASEBO | [Choromanski et al. (2019)](https://arxiv.org/abs/1903.04268) | [`GuidedES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/asebo.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| CMA-ES | [Hansen & Ostermeier (2001)](http://www.cmap.polytechnique.fr/~nikolaus.hansen/cmaartic.pdf) | [`CMA_ES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/cma_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | Sep-CMA-ES | [Ros & Hansen (2008)](https://hal.inria.fr/inria-00287367/document) | [`Sep_CMA_ES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/sep_cma_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | BIPOP-CMA-ES | [Hansen (2009)](https://hal.inria.fr/inria-00382093/document) | [`BIPOP_CMA_ES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/bipop_cma_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/06_restart_es.ipynb) | IPOP-CMA-ES | [Auer & Hansen (2005)](http://www.cmap.polytechnique.fr/~nikolaus.hansen/cec2005ipopcmaes.pdf) | [`IPOP_CMA_ES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/ipop_cma_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/06_restart_es.ipynb) @@ -53,16 +52,20 @@ state.best_member, state.best_fitness | MA-ES | [Bayer & Sendhoff (2017)](https://www.honda-ri.de/pubs/pdf/3376.pdf) | [`MA_ES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/ma_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | LM-MA-ES | [Loshchilov et al. (2017)](https://arxiv.org/pdf/1705.06693.pdf) | [`LM_MA_ES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/lm_ma_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | RmES | [Li & Zhang (2017)](https://ieeexplore.ieee.org/document/8080257) | [`RmES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/rm_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) -| GLD | [Golovin et al. (2019)](https://arxiv.org/pdf/1911.06317.pdf) | [`GLD`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/gld.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) -| Simulated Annealing | [Rasdi Rere et al. (2015)](https://www.sciencedirect.com/science/article/pii/S1877050915035759) | [`SimAnneal`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/sim_anneal.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) -| ESMC | [Merchant et al. (2021)](https://proceedings.mlr.press/v139/merchant21a.html) | [`ESMC`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/esmc.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) -| DES | [Lange et al. (2022)](https://arxiv.org/abs/2211.11260) | [`DES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/des.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| Simple Genetic | [Such et al. (2017)](https://arxiv.org/abs/1712.06567) | [`SimpleGA`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/simple_ga.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | SAMR-GA | [Clune et al. (2008)](https://journals.plos.org/ploscompbiol/article?id=10.1371/journal.pcbi.1000187) | [`SAMR_GA`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/samr_ga.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | GESMR-GA | [Kumar et al. (2022)](https://arxiv.org/abs/2204.04817) | [`GESMR_GA`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/gesmr_ga.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) -| Guided ES | [Maheswaranathan et al. (2018)](https://arxiv.org/abs/1806.10230) | [`GuidedES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/guided_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) -| ASEBO | [Choromanski et al. (2019)](https://arxiv.org/abs/1903.04268) | [`GuidedES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/asebo.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) -| CR-FM-NES | [Nomura & Ono (2022)](https://arxiv.org/abs/2201.11422) | [`CR-FM-NES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/cr_fm_nes.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) | MR15-GA | [Rechenberg (1978)](https://link.springer.com/chapter/10.1007/978-3-642-81283-5_8) | [`MR15_GA`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/mr15_ga.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| Simple Gaussian | [Rechenberg (1978)](https://link.springer.com/chapter/10.1007/978-3-642-81283-5_8) | [`SimpleES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/simple_es.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| DES | [Lange et al. (2022)](https://arxiv.org/abs/2211.11260) | [`DES`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/des.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| Particle Swarm Optimization | [Kennedy & Eberhart (1995)](https://ieeexplore.ieee.org/document/488968) | [`PSO`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/pso.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| Differential Evolution | [Storn & Price (1997)](https://www.metabolic-economics.de/pages/seminar_theoretische_biologie_2007/literatur/schaber/Storn1997JGlobOpt11.pdf) | [`DE`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/de.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| GLD | [Golovin et al. (2019)](https://arxiv.org/pdf/1911.06317.pdf) | [`GLD`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/gld.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| Simulated Annealing | [Rasdi Rere et al. (2015)](https://www.sciencedirect.com/science/article/pii/S1877050915035759) | [`SimAnneal`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/sim_anneal.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/01_classic_benchmark.ipynb) +| Population-Based Training | [Jaderberg et al. (2017)](https://arxiv.org/abs/1711.09846) | [`PBT`](https://github.com/RobertTLange/evosax/tree/main/evosax/strategies/pbt.py) | [![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/05_quadratic_pbt.ipynb) + + + ## Installation ⏳ @@ -89,7 +92,7 @@ In order to use JAX on your accelerators, you can find more details in the [JAX * 📓 [Quadratic-PBT](https://github.com/RobertTLange/evosax/blob/main/examples/05_quadratic_pbt.ipynb): PBT on toy quadratic problem as in [Jaderberg et al. (2017)](https://arxiv.org/abs/1711.09846). * 📓 [Restart-Wrappers](https://github.com/RobertTLange/evosax/blob/main/examples/06_restart_es.ipynb): Custom restart wrappers as e.g. used in (B)IPOP-CMA-ES. * 📓 [Brax Control](https://github.com/RobertTLange/evosax/blob/main/examples/07_brax_control.ipynb): Evolve Tanh MLPs on Brax tasks using the `EvoJAX` wrapper. -* 📓 [Indirect Encodings](https://github.com/RobertTLange/evosax/blob/main/examples/08_encodings.ipynb): Find out how many parameters we need to evolve a pendulum controller. + ## Key Features 💵 diff --git a/evosax/experimental/decodings/hyper.py b/evosax/experimental/decodings/hyper.py index 975dede..716f71e 100644 --- a/evosax/experimental/decodings/hyper.py +++ b/evosax/experimental/decodings/hyper.py @@ -5,8 +5,8 @@ from flax.core import unfreeze from typing import Union, Optional from .decoder import Decoder -from ...utils import ParameterReshaper -from ...networks import HyperNetworkMLP +from .hyper_networks import HyperNetworkMLP +from ...utils import ParameterReshaper, ravel_pytree class HyperDecoder(Decoder): @@ -45,7 +45,7 @@ def __init__( self.vmap_dict = self.hyper_reshaper.vmap_dict def reshape(self, x: chex.Array) -> chex.ArrayTree: - """Perform reshaping for random projection case.""" + """Perform reshaping for hypernetwork case.""" # 0. Reshape genome into params for hypernetwork x_params = self.hyper_reshaper.reshape(x) # 1. Project parameters to raw dimensionality using hypernetwork @@ -53,9 +53,17 @@ def reshape(self, x: chex.Array) -> chex.ArrayTree: return hyper_x def reshape_single(self, x: chex.Array) -> chex.ArrayTree: - """Reshape a single flat vector using random projection matrix.""" + """Reshape a single flat vector using hypernetwork.""" # 0. Reshape genome into params for hypernetwork x_params = self.hyper_reshaper.reshape_single(x) # 1. Project parameters to raw dimensionality using hypernetwork hyper_x = jax.jit(self.hyper_network.apply)(x_params) return hyper_x + + def flatten(self, x: chex.ArrayTree) -> chex.Array: + """Reshaping pytree parameters into flat array.""" + return jax.vmap(ravel_pytree)(x) + + def flatten_single(self, x: chex.ArrayTree) -> chex.Array: + """Reshaping pytree parameters into flat array.""" + return ravel_pytree(x) diff --git a/evosax/networks/hyper_networks.py b/evosax/experimental/decodings/hyper_networks.py similarity index 100% rename from evosax/networks/hyper_networks.py rename to evosax/experimental/decodings/hyper_networks.py diff --git a/evosax/experimental/decodings/random.py b/evosax/experimental/decodings/random.py index 46b100a..9ba17cd 100644 --- a/evosax/experimental/decodings/random.py +++ b/evosax/experimental/decodings/random.py @@ -32,6 +32,10 @@ def __init__( self.project_matrix = jax.random.rademacher( rng, (self.num_encoding_dims, self.base_reshaper.total_params) ) + print( + "RandomDecoder: Encoding parameters to optimize -" + f" {num_encoding_dims}" + ) def reshape(self, x: chex.Array) -> chex.ArrayTree: """Perform reshaping for random projection case.""" diff --git a/evosax/networks/__init__.py b/evosax/networks/__init__.py index 1d9e6a5..4f77b1c 100644 --- a/evosax/networks/__init__.py +++ b/evosax/networks/__init__.py @@ -1,7 +1,6 @@ from .mlp import MLP from .cnn import CNN, All_CNN_C from .lstm import LSTM -from .hyper_networks import HyperNetworkMLP # Helper that returns model based on string name @@ -18,5 +17,4 @@ "All_CNN_C", "LSTM", "NetworkMapper", - "HyperNetworkMLP", ] diff --git a/evosax/problems/__init__.py b/evosax/problems/__init__.py index 1726592..120ce38 100755 --- a/evosax/problems/__init__.py +++ b/evosax/problems/__init__.py @@ -1,17 +1,17 @@ -from .control_gym import GymFitness +from .control_gym import GymnaxFitness from .vision import VisionFitness from .bbob import BBOBFitness from .sequence import SequenceFitness ProblemMapper = { - "Gym": GymFitness, + "Gymnax": GymnaxFitness, "Vision": VisionFitness, "BBOB": BBOBFitness, "Sequence": SequenceFitness, } __all__ = [ - "GymFitness", + "GymnaxFitness", "VisionFitness", "BBOBFitness", "SequenceFitness", diff --git a/evosax/problems/control_gym.py b/evosax/problems/control_gym.py index 678d072..5bc6583 100644 --- a/evosax/problems/control_gym.py +++ b/evosax/problems/control_gym.py @@ -4,7 +4,7 @@ import chex -class GymFitness(object): +class GymnaxFitness(object): def __init__( self, env_name: str = "CartPole-v1", @@ -46,7 +46,7 @@ def __init__( # Keep track of total steps executed in environment self.total_env_steps = 0 - def set_apply_fn(self, map_dict, network_apply, carry_init=None): + def set_apply_fn(self, network_apply, carry_init=None): """Set the network forward function.""" self.network = network_apply # Set rollout function based on model architecture @@ -56,9 +56,7 @@ def set_apply_fn(self, map_dict, network_apply, carry_init=None): else: self.single_rollout = self.rollout_ffw self.rollout_repeats = jax.vmap(self.single_rollout, in_axes=(0, None)) - self.rollout_pop = jax.vmap( - self.rollout_repeats, in_axes=(None, map_dict) - ) + self.rollout_pop = jax.vmap(self.rollout_repeats, in_axes=(None, 0)) # pmap over popmembers if > 1 device is available - otherwise pmap if self.n_devices > 1: self.rollout_map = self.rollout_pmap diff --git a/evosax/problems/sequence.py b/evosax/problems/sequence.py index 5425df4..a9c2175 100644 --- a/evosax/problems/sequence.py +++ b/evosax/problems/sequence.py @@ -45,11 +45,11 @@ def __init__( else: self.n_devices = n_devices - def set_apply_fn(self, map_dict, network, carry_init): + def set_apply_fn(self, network, carry_init): """Set the network forward function.""" self.network = network self.carry_init = carry_init - self.rollout_pop = jax.vmap(self.rollout_rnn, in_axes=(None, map_dict)) + self.rollout_pop = jax.vmap(self.rollout_rnn, in_axes=(None, 0)) # pmap over popmembers if > 1 device is available - otherwise pmap if self.n_devices > 1: self.rollout = self.rollout_pmap diff --git a/evosax/problems/vision.py b/evosax/problems/vision.py index 4f4971a..72989ff 100644 --- a/evosax/problems/vision.py +++ b/evosax/problems/vision.py @@ -25,10 +25,10 @@ def __init__( else: self.n_devices = n_devices - def set_apply_fn(self, map_dict, network): + def set_apply_fn(self, network): """Set the network forward function.""" self.network = network - self.rollout_pop = jax.vmap(self.rollout_ffw, in_axes=(None, map_dict)) + self.rollout_pop = jax.vmap(self.rollout_ffw, in_axes=(None, 0)) # pmap over popmembers if > 1 device is available - otherwise pmap if self.n_devices > 1: self.rollout = self.rollout_pmap diff --git a/evosax/strategies/de.py b/evosax/strategies/de.py index 77dc9dd..0c6ff7c 100755 --- a/evosax/strategies/de.py +++ b/evosax/strategies/de.py @@ -38,7 +38,7 @@ def __init__( ): """Differential Evolution (Storn & Price, 1997) Reference: https://tinyurl.com/4pje5a74""" - assert popsize > 6 + assert popsize > 6, "DE requires popsize > 6." super().__init__(popsize, num_dims, pholder_params, **fitness_kwargs) self.strategy_name = "DE" diff --git a/evosax/strategies/snes.py b/evosax/strategies/snes.py index e80e02d..77d5b87 100644 --- a/evosax/strategies/snes.py +++ b/evosax/strategies/snes.py @@ -4,6 +4,7 @@ from typing import Tuple, Optional, Union from ..strategy import Strategy from flax import struct +from flax import linen as nn @struct.dataclass @@ -21,6 +22,7 @@ class EvoParams: lrate_mean: float = 1.0 lrate_sigma: float = 1.0 sigma_init: float = 1.0 + temperature: float = 0.0 init_min: float = 0.0 init_max: float = 0.0 clip_min: float = -jnp.finfo(jnp.float32).max @@ -38,6 +40,17 @@ def get_weight(i): return weights_norm - use_baseline * (1 / popsize) +def get_temp_weights( + popsize: int, temperature: float, use_baseline: bool = True +): + """Get weights based on original discovered weights (Lange et al, 2022).""" + ranks = jnp.arange(popsize) + ranks /= ranks.size - 1 + ranks = ranks - 0.5 + weights = nn.softmax(-temperature * ranks) + return weights + + class SNES(Strategy): def __init__( self, @@ -45,6 +58,7 @@ def __init__( num_dims: Optional[int] = None, pholder_params: Optional[Union[chex.ArrayTree, chex.Array]] = None, sigma_init: float = 1.0, + temperature: float = 0.0, # good values tend to be between 12 and 20 **fitness_kwargs: Union[bool, int, float] ): """Separable Exponential Natural ES (Wierstra et al., 2014) @@ -55,6 +69,7 @@ def __init__( # Set core kwargs es_params self.sigma_init = sigma_init + self.temperature = temperature @property def params_strategy(self) -> EvoParams: @@ -62,7 +77,11 @@ def params_strategy(self) -> EvoParams: lrate_sigma = (3 + jnp.log(self.num_dims)) / ( 5 * jnp.sqrt(self.num_dims) ) - params = EvoParams(lrate_sigma=lrate_sigma, sigma_init=self.sigma_init) + params = EvoParams( + lrate_sigma=lrate_sigma, + sigma_init=self.sigma_init, + temperature=self.temperature, + ) return params def initialize_strategy( @@ -75,7 +94,12 @@ def initialize_strategy( minval=params.init_min, maxval=params.init_max, ) - weights = get_recombination_weights(self.popsize) + use_des_weights = params.temperature > 0.0 + weights = jax.lax.select( + use_des_weights, + get_temp_weights(self.popsize, params.temperature), + get_recombination_weights(self.popsize), + ) state = EvoState( mean=initialization, sigma=params.sigma_init * jnp.ones(self.num_dims), diff --git a/evosax/strategy.py b/evosax/strategy.py index 469110a..c6eb590 100755 --- a/evosax/strategy.py +++ b/evosax/strategy.py @@ -119,7 +119,9 @@ def tell( state = self.tell_strategy(x, fitness_re, state, params) # Check if there is a new best member & update trackers - best_member, best_fitness = get_best_fitness_member(x, fitness, state) + best_member, best_fitness = get_best_fitness_member( + x, fitness, state, self.fitness_shaper.maximize + ) return state.replace( best_member=best_member, best_fitness=best_fitness, diff --git a/evosax/utils/__init__.py b/evosax/utils/__init__.py index a8b4203..3c09aab 100755 --- a/evosax/utils/__init__.py +++ b/evosax/utils/__init__.py @@ -2,7 +2,7 @@ from .es_logger import ESLog # Import additional utilities for reshaping flat parameters into net dict -from .reshape_params import ParameterReshaper +from .reshape_params import ParameterReshaper, ravel_pytree # Import additional utilities for reshaping fitness from .reshape_fitness import FitnessShaper @@ -35,6 +35,7 @@ "get_best_fitness_member", "ESLog", "ParameterReshaper", + "ravel_pytree", "FitnessShaper", "GradientOptimizer", "SGD", diff --git a/evosax/utils/helpers.py b/evosax/utils/helpers.py index 87e93c2..21dd54a 100644 --- a/evosax/utils/helpers.py +++ b/evosax/utils/helpers.py @@ -5,19 +5,25 @@ def get_best_fitness_member( - x: chex.Array, fitness: chex.Array, state + x: chex.Array, fitness: chex.Array, state, maximize: bool = False ) -> Tuple[chex.Array, float]: """Check if fitness improved & replace in ES state.""" - best_in_gen = jnp.argmin(fitness) + fitness_min = jax.lax.select(maximize, -1 * fitness, fitness) + max_and_later = maximize and state.gen_counter > 0 + best_fit_min = jax.lax.select( + max_and_later, -1 * state.best_fitness, state.best_fitness + ) + best_in_gen = jnp.argmin(fitness_min) best_in_gen_fitness, best_in_gen_member = ( - fitness[best_in_gen], + fitness_min[best_in_gen], x[best_in_gen], ) - replace_best = best_in_gen_fitness < state.best_fitness + replace_best = best_in_gen_fitness < best_fit_min best_fitness = jax.lax.select( - replace_best, best_in_gen_fitness, state.best_fitness + replace_best, best_in_gen_fitness, best_fit_min ) best_member = jax.lax.select( replace_best, best_in_gen_member, state.best_member ) + best_fitness = jax.lax.select(maximize, -1 * best_fitness, best_fitness) return best_member, best_fitness diff --git a/evosax/utils/visualizer_2d.py b/evosax/utils/visualizer_2d.py index 2171213..e9b8c6c 100644 --- a/evosax/utils/visualizer_2d.py +++ b/evosax/utils/visualizer_2d.py @@ -1,5 +1,6 @@ """Fitness landscape visualizer and evaluation animator.""" import chex +import jax import jax.numpy as jnp import numpy as np import matplotlib.cm as cm @@ -18,11 +19,13 @@ class BBOBVisualizer(object): def __init__( self, X: chex.Array, + fitness: chex.Array, fn_name: str = "Rastrigin", title: str = "", use_3d: bool = False, ): self.X = X + self.fitness = fitness self.title = title self.fn_name = fn_name self.use_3d = use_3d @@ -33,22 +36,40 @@ def __init__( self.ax = self.fig.add_subplot(1, 1, 1, projection="3d") self.fn_name = fn_name self.fn = BBOB_fns[self.fn_name] - self.R = jnp.array(get_rotation(2, 0, b"R")) - self.Q = jnp.array(get_rotation(2, 0, b"Q")) + + rng = jax.random.PRNGKey(0) + rng_q, rng_r = jax.random.split(rng) + self.R = get_rotation(rng_r, 2) + self.Q = get_rotation(rng_q, 2) self.global_minima = [] + # Set boundaries for evaluation range of black-box functions self.x1_lower_bound, self.x1_upper_bound = -5, 5 self.x2_lower_bound, self.x2_upper_bound = -5, 5 + # Set meta-data for rotation/azimuth + self.interval = 50 # Delay between frames in milliseconds. + try: + self.num_frames = X.shape[0] + self.static_frames = int(0.2 * self.num_frames) + self.azimuths = jnp.linspace( + 0, 90, self.num_frames - self.static_frames + ) + self.angles = jnp.linspace( + 0, 90, self.num_frames - self.static_frames + ) + except Exception: + pass + def animate(self, save_fname: str): """Run animation for provided data.""" ani = animation.FuncAnimation( self.fig, self.update, - frames=self.X.shape[0], + frames=self.num_frames, init_func=self.init, blit=False, - interval=10, + interval=self.interval, ) ani.save(save_fname) @@ -59,7 +80,7 @@ def init(self): (self.scat,) = self.ax.plot( self.X[0, :, 0], self.X[0, :, 1], - jnp.ones(X.shape[1]) * 0.1, + self.fitness[0, :], marker="o", c="r", linestyle="", @@ -86,7 +107,9 @@ def update(self, frame): # Plot sample points self.scat.set_data(self.X[frame, :, 0], self.X[frame, :, 1]) if self.use_3d: - self.scat.set_3d_properties(jnp.ones(X.shape[1]) * 0.1) + self.scat.set_3d_properties(self.fitness[frame, :]) + if frame < self.num_frames - self.static_frames: + self.ax.view_init(self.azimuths[frame], self.angles[frame]) self.ax.set_title( f"{self.fn_name}: {self.title} - Generation {frame + 1}", fontsize=15, @@ -190,18 +213,26 @@ def plot_contour_3d(self, save: bool = False): rng = jax.random.PRNGKey(42) - for fn_name in [ - "BuecheRastrigin", - ]: # BBOB_fns.keys(): - print(f"Start 2d/3d - {fn_name}") - visualizer = BBOBVisualizer(None, fn_name, "") - visualizer.plot_contour_2d(save=True) - visualizer.plot_contour_3d(save=True) - - # # Test animations - # # All solutions from single run (10 gens, 16 pmembers, 2 dims) - # X = jax.random.normal(rng, shape=(10, 16, 2)) - # visualizer = BBOBVisualizer(X, "Ackley", "Test Strategy", use_3d=True) - # visualizer.animate("Ackley_3d.gif") - # visualizer = BBOBVisualizer(X, "Ackley", "Test Strategy", use_3d=False) - # visualizer.animate("Ackley_2d.gif") + # for fn_name in [ + # "BuecheRastrigin", + # ]: # BBOB_fns.keys(): + # print(f"Start 2d/3d - {fn_name}") + # visualizer = BBOBVisualizer(None, None, fn_name, "") + # visualizer.plot_contour_2d(save=True) + # visualizer.plot_contour_3d(save=True) + + # Test animations + # All solutions from single run (10 gens, 16 pmembers, 2 dims) + X = jax.random.normal(rng, shape=(50, 16, 2)) + + def sphere(x): + return jnp.sum(x ** 2) + + fitness = jax.vmap(jax.vmap(sphere))(X) + print(fitness.shape) + visualizer = BBOBVisualizer( + X, fitness, "Sphere", "Test Strategy", use_3d=True + ) + visualizer.animate("Sphere_3d.gif") + # visualizer = BBOBVisualizer(X, None, "Sphere", "Test Strategy", use_3d=False) + # visualizer.animate("Sphere_2d.gif") diff --git a/examples/00_getting_started.ipynb b/examples/00_getting_started.ipynb index 9a4acaa..35187d4 100644 --- a/examples/00_getting_started.ipynb +++ b/examples/00_getting_started.ipynb @@ -35,15 +35,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "9dee8f4d-9ce8-4f5b-8d9a-ccde409dd873", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "EvoParams(mu_eff=DeviceArray(3.1672993, dtype=float32), c_1=DeviceArray(0.14227484, dtype=float32), c_mu=DeviceArray(0.1547454, dtype=float32), c_sigma=DeviceArray(0.50822735, dtype=float32), d_sigma=DeviceArray(1.5082273, dtype=float32), c_c=DeviceArray(0.60908335, dtype=float32), chi_n=DeviceArray(1.2542727, dtype=float32, weak_type=True), c_m=1.0, sigma_init=1.0, init_min=-3, init_max=3, clip_min=-3.4028235e+38, clip_max=3.4028235e+38)" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "import jax\n", "import jax.numpy as jnp\n", "from evosax import CMA_ES\n", - "from evosax.problems import ClassicFitness\n", + "from evosax.problems import BBOBFitness\n", "\n", "# Instantiate the evolution strategy instance\n", "strategy = CMA_ES(num_dims=2, popsize=10)\n", @@ -70,13 +81,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "id": "969621cb", "metadata": {}, "outputs": [], "source": [ "# Instantiate helper class for classic evolution strategies benchmarks\n", - "evaluator = ClassicFitness(\"rosenbrock\", num_dims=2)" + "evaluator = BBOBFitness(\"RosenbrockOriginal\", num_dims=2)" ] }, { @@ -89,10 +100,25 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "e8982ce5-91b0-4ccc-ba46-2c69d4f11287", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "EvoState(p_sigma=DeviceArray([1.4210665 , 0.17011967], dtype=float32), p_c=DeviceArray([1.5021834, 0.1798304], dtype=float32), C=DeviceArray([[1.6521862, 0.2126719],\n", + " [0.2126719, 0.6064514]], dtype=float32), D=None, B=None, mean=DeviceArray([-0.7851852, 1.9345263], dtype=float32), sigma=DeviceArray(1.0486844, dtype=float32), weights=DeviceArray([ 0.45627266, 0.27075312, 0.16223112, 0.08523354,\n", + " 0.02550957, -0.09313666, -0.25813875, -0.4010702 ,\n", + " -0.5271447 , -0.639922 ], dtype=float32), weights_truncated=DeviceArray([0.45627266, 0.27075312, 0.16223112, 0.08523354, 0.02550957,\n", + " 0. , 0. , 0. , 0. , 0. ], dtype=float32), best_member=DeviceArray([0.7087054, 2.3278952], dtype=float32), best_fitness=DeviceArray(17.166704, dtype=float32), gen_counter=DeviceArray(1, dtype=int32, weak_type=True))" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "# Ask for a set of candidate solutions to evaluate\n", "x, state = strategy.ask(rng, state, es_params)\n", @@ -113,7 +139,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "57b3e83c-e81f-48bd-aa8c-e90b7542502a", "metadata": {}, "outputs": [], @@ -127,10 +153,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "791a2d65-7404-40a2-9c4b-4a1e1ac12dfa", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(
,\n", + " )" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAADgCAYAAAAEwQ17AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAA93klEQVR4nO2dd7xU1bX4v2tmbm+0S5EOighSlAuIGEVjjYrdJzEKL1GeiSX6iy1qfKjEJz7b0yRGsaA+nygYwaiJBUXFqIBKF0WQ3tut3Drr98c5Mwy3zi0zd+bO+n4+5zPn7H32PmvPnbtmzdp7ryWqimEYhpE4eFpbAMMwDCO6mOI3DMNIMEzxG4ZhJBim+A3DMBIMU/yGYRgJhil+wzCMBMMUv2GEICLzReSqKDxnhohMjfRzDKM2TPEbYSEiKSLyrIhsEJFCEVkiImeF1I8TEb+IFLnHZhF5TURG1tNnHxHRkDbrReT26IwovhCRM0TkE/e93yUiH4vIeLdukvs+PlqtzXlu+Yxq5Znu+/2PMJ47X0RKQ/5GRSLy95D6O0Tkx5C/+astNGQjgpjiN8LFB2wCTgJygLuA10SkT8g9W1U1E8gCjgNWA5+KyE8b6Lud2+5i4A8iclpLCx8JRMQXpedcDMwCXgR6AF2Au4FzQ25bC1xaTaaJwPe1dHkRUAacJiJdwxDhOlXNDDnOdeWaCFwBnOr+/fKAeY0bndEamOI3wkJVi1V1iqquV1W/qr4F/AiMqOVeVdXNqno38AwwLcxnLAZWAsMBRMQjIne5vzJ2isiLIpLj1qWKyP+KyB4R2S8ii0Ski1uX4/462SYiW0Rkqoh43bpJIrJARB4SkX2utXpWNVH6i8hCESkQkbki0sFtG/iF8isR2Qh8WJ+MbpsTRORfroybRGRS9XGLSJaIfCQij4uIVKsT4BHgPlV9RlXz3ff/Y1W9OuTW7cBy4Ay3XQfgeODNWt7qicBfgWXAL8L409TFSOBdVV0LoKrbVfXpZvRnRAlT/EaTcJXsABxFXR9/A44VkYww+jwOOBr4wS2a5B4nA/2ATOBPbt1EnF8ePYGOwDXAAbduBlAJHA4cA5wOhPrtRwPfAZ2AB4FnqyncK4FfAt3cfh6vJupJwFE4SrZOGUWkN/AP4AkgF+cLbUm1MXfEsZI/U9UbtGYMlSPdMc6mYV50ZQe4DJiLY9mHPq83MA542T2upOl8AVwpIreISF7gy9WIA1TVDjsadQBJwAfAUyFl44DNtdw7EFCgey11fdy6/ThKW4GHAHHr5wG/Cbn/SKACx+30S+BfwNBqfXbBUXZpIWUTgI/c80nADyF16e5zu7rX84EHQuoHAeWAN0TefiH19cn4e+CNOt7DGcBzwArglnre67HuM1PruWcSsABIA3bgfCF+4badCswIufcuYIl73h2oAo6pp+/5QIn7Nwoc94XUX+5+FoqBPcBtrf35tKPhwyx+o1GIiAd4CUcZXhdGk+4cVO510QnHUv4dzhdIklt+GLAh5L4NOAq1iyvDu8BMEdkqIg+KSBLQ222/zXWv7AeeAjqH9LM9cKKqJe5pZkj9pmrPTHJlrK2+Phl74vje6+JsHGX913ru2eO+dqvnHgBU9QDwNo5y76iqn9Vy25U4lj6qugX4GOfXEyLy15AJ3DtC2tygqu1Cjj+EPPNlVT0VaIfzq+s+ETmjIVmN1sUUvxE2rjvkWRyldpGqVoTR7ALga1Utru8mVa1S1UeAUuA3bvFWHEUeoBeO62WHqlao6j2qOgjHl30OjlLbhGPxdwpRVNmqOjj8kdKz2jMrgN2h4oac1ymjK0v/ep4zHfgn8E49rrDv3H4uCktyx93zO+B/q1eIyPHAEcDvRWS7iGzHcXv9XER8qnqNHpzAvT/M5wHg/j1m4cwbHN2Ytkb0McVvNIYncXzb57rWZa2IQ3cR+U8c3/oddd1bCw8At4pIKvAKcJOI9BWRTOB+4FVVrRSRk0VkiOtXLsBRzn5V3Qa8BzwsItnu5Gt/ETmpETL8QkQGiUg6cC8wW1Wr6ri3ThlxLOtTReRSEfGJSEcRGV6t/XU4yv3vIpJWvXNVVeD/4ax2+veQMZ0gIrVNpH4MnIYzr1CdicD7OO6r4e5xNM6vjuoT3A3iTpSf7U5Oe9xJ8sHAl43ty4gupviNsHAnBf8DR1lsD3EJXB5y22EiUgQUAYuAIcA4VX2vEY96G9gHXI3jA38J+ARnBVEpcL17X1ecCc8C4FschfeSW3clkAyscvuaTRiukhBewvHBbwdSgRvqubdOGVV1I/AzHAt8L87E7rDQxq5inwxsBua6X3hUu2c28G848xpbcX5NTMWZvK1+r6rqPFXdG1ru9nsp8IQ6q28Cx4+u/BPrGeOf5NB1/F+55QU4X+obcVx5DwK/VtUF9fRlxACBSTTDMAwjQTCL3zAMI8GImOIXZ4PNQhFZKiIrReQet7yviHwpIj+IyKsikhwpGQzDMIyaRNLiLwNOUdVhOH7hM90NOtOAR1X1cBz/668iKINhGIZRjYgpfneSqci9THIPBU7h4C7EF4DzIyWDYRiGUZOI+vhFxCsiS4CdOMvI1gL73aVu4Kxk6B5JGQzDMIxDiajidzflDMeJKDgKZ/t+WIjIZBFZLCKLBw8erDi/Fpp/zJqkPJHXqDY3fHiDDnlhiA55YYhuK9rWcrK04FHy9Tf67cCjtOjTBa0uix122BEzR61EZVWPqu4HPgLGAO3kYOjYHsCWOto8rap5qpqXllZjX0vT8figzr04tVNSWRI8L60qbTlZWhBvdhYA/sKCVpbEMIxYJ5KrenJFpJ17noazm/BbnC+Ai93bJlLLJpSI4vGBv7Lh+0I4UHFwk2pZVVk9d7YenqxsAKoKTPEbhlE/kUwk0Q14wd1S7wFeU9W3RGQVTmCtqcA3OLFfoofHC/7GW/xpvjQOVB6gtDJGLf6cgOIvbGVJDMOIdSKm+FV1GU4s9Orl63D8/a1DEyz+kooS2qe050DlgZi1+CUlBUlKMlePYRgNEpXUcTFFUxR/ZQndM7uztXhr7Cp+ETzZ2VTlm+I3Wp+Kigo2b95MaWls/kJua6SmptKjRw+SkpIavplEVPzibZrFn9oeIGZdPQDe7GyqzOI3YoDNmzeTlZVFnz59qJZN0mhhVJU9e/awefNm+vbtG1abxIvV4/GB3x/27RX+Csr95XRI7QDE7uQugCc7C7/5+I0YoLS0lI4dO5rSjwIiQseOHRv16yoBFX/jLP4Dlc6KnvYprsUfo8s5AbxZ2baqx4gZTOlHj8a+1wmo+Bvn4y+pcNbwB1w9ZZWxa/F7s7Pxm+I3DPbs2cPw4cMZPnw4Xbt2pXv37sHr8vLyRve3evVqxowZQ0pKCg899FAEJI4uiefjb6zidzdvBVw9sWzxe7KzqCo0V49hdOzYkSVLlgAwZcoUMjMzufnmm5vcX4cOHXj88ceZM2dOywjYyiSmxY+G7ecPbN5ql9IOiG0ff8DVY8l1DKMm8+bN45hjjmHIkCH88pe/pKzM+V/u06cPt956K0OGDGHUqFH88MMPNdp27tyZkSNHhr1qJtZJQIvf67z6K8HTcCqAgMWfmZxJijcltl09OdlQWYkeOICkp7e2OIYBwD1/X8mqrS3rghx0WDb/ee7gsO8vLS1l0qRJzJs3jwEDBnDllVfy5JNPcuONNwKQk5PD8uXLefHFF7nxxht56623WlTeWCMBLf4QxR8GAR9/mi+NFG9KcLI3FgmGbTB3j2EcQlVVFX379mXAgAEATJw4kU8++SRYP2HChODr559/3ioyRpMEtPjdIYcZqC1g8af70kn1psa2q8cN1FaVn09Sly6tLI1hODTGMm8tQlfFJMJqpAS0+F3F30iLPz0pnRRfSoxP7joWv98sfsM4BK/Xy/r164P++5deeomTTjopWP/qq68GX8eMGdMqMkaTxLX4wwzUFrD4A66emPbxZ1uETsOojdTUVJ5//nkuueQSKisrGTlyJNdcc02wft++fQwdOpSUlBReeeWVGu23b99OXl4eBQUFeDweHnvsMVatWkW2+z8XbySg4m+ajz8uXD1Zbkx+U/yGEWTKlCnB82+++abWe2655RamTZtWZx9du3Zl8+bNLS1aq2GungYoqSwhyZNEkjcp9l09OTmAhWY2DKN+EtDib7yPPz3JWRqZ6k0lvyw/UpI1G29mJoAFajOMRrB+/frWFiHqJJ7FLwFXT/g+/nSfo/hTvLFt8UtSEpKejt9CMxuGUQ+Jp/g9jVP8ByoPHFT8vpSY9vFDIDSzuXoMw6ibBFT8TXf1pPnSYnpVDzgTvJaFyzCM+ohksvWeIvKRiKwSkZUi8lu3fIqIbBGRJe7xs0jJUCtNmNyNF1cPUG8WrqqiYnY+9hh+y4pkGAlNJC3+SuB3qjoIOA64VkQGuXWPqupw93gngjLUpAkWf1pSGkDML+eE+l09+W/OZc9fn+LA119HWSrDiC4tHZYZ4Mwzz2TYsGEMHjyYa665hqqqmu7iKVOmICKHBHp77LHHEBEWL17c5PG0NBFT/Kq6TVW/ds8LgW+B7pF6Xtg0YQNXdR+/X8PP4BVtvNlZda7jL3z/fQCqioujKZJhRJ1AWOYlS5ZwzTXXcNNNNwWvk5MbDs5YG6+99hpLly5lxYoV7Nq1i1mzZtV635AhQ5g5c2bwetasWQweHFthK6Li4xeRPsAxwJdu0XUiskxEnhOR9tGQIYjHHXITfPwp3hQgtkMze+rIwlW1fz8lCxcB4C8yxW8kHs0JywwEd+lWVlZSXl5eZ0yf888/n7lz5wKwdu1acnJy6NSpU7D+vffeY8yYMRx77LFccsklFBUVAXDvvfcycuRIjj76aCZPnhwMrz5u3Dhuu+02Ro0axYABA/j000+b/V5EfB2/iGQCrwM3qmqBiDwJ3Aeo+/ow8Mta2k0GJgP06tWr5QRqho8/1ZsKOFm40nxpLSdTC+LNzsZfVIT6/Yjn4Pd64fz54P409bsfNMOICv+4HbYvb9k+uw6Bsx4I+/aWCst8xhlnsHDhQs466ywuvvjiWu/Jzs6mZ8+erFixgrlz5/Jv//ZvPP/88wDs3r2bqVOn8sEHH5CRkcG0adN45JFHuPvuu7nuuuu4++67Abjiiit46623OPfccwHny2bhwoW888473HPPPXzwwQdhj702Imrxi0gSjtJ/WVX/BqCqO1S1SlX9wHRgVG1tVfVpVc1T1bzc3NyWE6oR0Tn96neWcyYddPVA7GfhQrWGci/84AO8uY7V4S82xW8kFi0Vlvndd99l27ZtlJWV8eGHH9Z532WXXcbMmTOZM2cOF1xwQbD8iy++YNWqVYwdO5bhw4fzwgsvsGHDBgA++ugjRo8ezZAhQ/jwww9ZuXJlsN2FF14IwIgRI1pkw1nELH5xfgc9C3yrqo+ElHdT1W3u5QXAikjJUCuNsPhLKx0FX8Pij2FXjzfrYKC2QNA2/4EDFC/4jHYXXcT+2bPN4jeiSyMs89aieljmqqoqRowYAcD48eO59957g/Wpqamcd955zJ07l9NOO63W/s455xxuueUW8vLyDgnkpqqcdtppNQLBlZaW8pvf/IbFixfTs2dPpkyZQmnI6ruUFMfo9Hq9VFaGnzq2LiJp8Y8FrgBOqbZ080ERWS4iy4CTgZsiKENNGrGBKzQWPxz08Qe+EGIRb46r7EP8/EULFqClpWSddiqejAyqTPEbCUZjwzJ7vd7gZPC9995LUVER27Y59mplZSVvv/02AwcOrPN56enpTJs2jTvvvPOQ8uOOO47PPvssKEdxcTHff/99UMl36tSJoqIiZs+e3XKDr4WIWfyqugCobfYjuss3q9MIiz+QbzcYq8cX+xZ/MAtXSKC2wvffx5uTQ3peHp7MTPzFJa0lnmG0Cs0Ny1xcXMz48eMpKyvD7/dz8sknH9K+Ni677LIaZbm5ucyYMYMJEyYEJ5enTp3KgAEDuPrqqzn66KPp2rUrI0eObOaI68eCtNVDXRZ/LCv+YBauAieYnFZUUDT/Y7JOOQXx+fBkZpirx0goWiIsc5cuXVi0aFGjnhXK/Pnzg+ennHJKrX1NnTqVqVOn1tu2U6dOLeLjT7yQDRJ+PP5gEpbABi7X4o9pV0+1LFzFCxfiLygg67RTnfqMTFP8hpHgJLDFH4aPv+JQiz8eJnc92Ye6ego/+ABJSyNj7FinPjOTih3bW00+w4g1LCxzItCUyd1qG7hiejlnRgaIUFWQj/r9FH0wj8wTTsCT6nxpeTIzbQOXYSQ4Caj4G+Hjr27x+w5u4IpVxOPBk52Nv6CQ0mXLqNy1i6zTDy45Mx+/YRim+OshHi1+cEIzVxUWUPD+++DzkRmybM2bYYrfMBKdBPbxN97ij4d1/ODs3vXnF1C4dCkZo0cHJ3zBcfVoeTlaXo40MViVYRjxTQJa/I3z8XvEE1T48bCOH8CbncOBZcuo2LAxuJongCfDzctrETqNNs6OHTv4+c9/Tr9+/RgxYgRjxozhjTfeaPHnrF69mjFjxpCSksJDDz3U4v1HggRW/OFZ/Om+9OB2bo94SPIkxYerZ98+ECHzlFMOqfO4Cdn9pviNNoyqcv7553PiiSeybt06vvrqK2bOnMnmzZtb/FkdOnTg8ccf5+abb27xviNFAir+RuzcDcm3GyDVmxrTk7vgBmoD0oYNI6lz50PrMjMAi9BptG0+/PBDkpOTD9ld27t3b66//nrACdp2yy23MHLkSIYOHcpTTz0FOJulxo0bx8UXX8zAgQO5/PLLg+GR66Jz586MHDmSpKSkyA2ohUlcH38Y0TlDs28FiI+E6zkANdw8AN6AxW+K34gS0xZOY/Xe1S3a58AOA7lt1G111q9cuZJjjz22zvpnn32WnJwcFi1aRFlZGWPHjuX0008HnN29K1eu5LDDDmPs2LF89tlnnHDCCS0qf2uTwBZ/eD7+6hZ/POTd9bZ3cttknVpT8XsyHIvfArUZicS1117LsGHDgjFw3nvvPV588UWGDx/O6NGj2bNnD2vWrAFg1KhR9OjRA4/Hw/Dhw9vkBq/Es/gl/AxcJZUlNRKupPnSYt7V0+7ii0gZcATJvXvXqAv6+G0TlxEl6rPMI8XgwYN5/fXXg9d//vOf2b17N3l5eYAzB/DEE09wxhlnHNJu/vz5wRDI0HJhkGONxLP4RRyrP9zJ3aT4s/h9HTqQNW5crXWBVT02uWu0ZU455RRKS0t58skng2UlJQej0p5xxhk8+eSTVFRUAPD9999TnED/E4ln8UP4ir+yhO6+Q/PDp3hj38dfH16b3DUSABFhzpw53HTTTTz44IPk5uYGUx0CXHXVVaxfv55jjz0WVSU3N5c5c+bU2+fdd99NXl4e48ePP6R8+/bt5OXlUVBQgMfj4bHHHmPVqlWHJGCJNRJT8Ys37CBt1S3+VF8qReXxqzQlPR1ELP2i0ebp1q0bM2fOrLXO4/Fw//33c//99x9SPm7cOMaF/Fr+05/+FDwPzcIVSteuXSOyTDSSJJ6rBxpl8cfj5G59iAiezEyb3DWMBCZBFX/DFr+qcqDiQE2L35sa8yEbGsIidBpGYhO24heR9IbvihPCsPgr/BVUamVNi98X3xY/gCcj3Xz8hpHANKj4ReR4EVkFrHavh4nIX8Jo11NEPhKRVSKyUkR+65Z3EJH3RWSN+9q+2aNoLGEo/mCAtlpW9cTz5C5YFi7DSHTCsfgfBc4A9gCo6lLgxDDaVQK/U9VBwHHAtSIyCLgdmKeqRwDz3Ovo4vE16Oqpnm83QDyEbGgIJ+G6uXoMI1EJy9WjqpuqFTW4JEZVt6nq1+55IfAt0B04D3jBve0F4PxwhW0xPJ6wLf7aQjaUVpU2GL8jlvFkZlJlq3oMI2EJR/FvEpHjARWRJBG5GUeJh42I9AGOAb4EuqjqNrdqO9CljjaTRWSxiCzetWtXYx7XMOG4euqx+AHK/eUtK1MUcbJwmcVvtG2iFZZ5xowZiAgffPBBsGzOnDmICLNnz27x57UE4Sj+a4Brcaz1LcBw9zosRCQTeB24UVULQuvUMZtrNZ1V9WlVzVPVvNzc3HAfFx7NUPzxkoylPryZ5uM32jbRDMsMMGTIkEP2DLzyyisMGzYsIs9qCepV/CLiBf5HVS9X1S6q2llVf6Gqe8LpXESScJT+y6r6N7d4h4h0c+u7ATubIX/T8PhA/fXeUtfkbrwkY6kPT0YG/uJi1F//e2AY8Uo0wzID/OQnP2HhwoVUVFRQVFTEDz/8wPDhw4P1X331FSeddBIjRozgjDPOYNs2x+kxffp0Ro4cybBhw7jooouCYSUmTZrEDTfcwPHHH0+/fv1a/JdDvTt3VbVKRHqLSLKqNsq3IU72kmeBb1X1kZCqN4GJwAPu69xGytx8PN6mu3riIOF6Q3gyMkEVf8mBYAgHw4gU2++/n7JvWzYsc8pRA+l6xx111kc7LLOIcOqpp/Luu++Sn5/P+PHj+fHHHwGoqKjg+uuvZ+7cueTm5vLqq69y55138txzz3HhhRdy9dVXA3DXXXfx7LPPBr+ctm3bxoIFC1i9ejXjx4/n4osvbtR7VB/hhGxYB3wmIm8CQcdwNWVeG2OBK4DlIrLELbsDR+G/JiK/AjYAlzZW6GYThqvnQOUBoPblnBD7Cdfr42AWriJT/EZCcO2117JgwQKSk5NZtGgR7733HsuWLQta0vn5+axZs4bk5ORgWGYgGJY5nHj8l112GY8//jj5+fk8/PDDwXAQ3333HStWrOC0004DnF8b3bp1A2DFihXcdddd7N+/n6KiokOihZ5//vl4PB4GDRrEjh07WvT9CEfxr3UPD5AVbsequgCQOqp/Gm4/EaEx6/jrmNyNa1dPIFCbLek0okB9lnmkaI2wzKNGjWL58uWkp6czYMCAYLmqMnjwYD7//PMabSZNmsScOXMYNmwYM2bMYP78+cG6UDlaehVhg5O7qnqPqt4DPAw8HHIdv4QRpC3g6qkejz/F1zYmd8EidBptl9YKy/zAAw/UCPx25JFHsmvXrqDir6ioYOXKlQAUFhbSrVs3KioqePnll5v9/HBp0OIXkaOBl4AO7vVu4EpVXRlh2SJHGD7+AxUHSPWm4g0kZ3dpGxa/KX6jbRPNsMyhnHXWWTXKkpOTmT17NjfccAP5+flUVlZy4403MnjwYO677z5Gjx5Nbm4uo0ePprCwsFnjDhtVrfcA/gWcHHI9DvhXQ+1a8hgxYoS2KC+cpzr9p/Xect/n9+mJM0+sUb5q9yo9esbR+sH6D1pWpihyYPVqXXXkQM1/993WFsVoo6xataq1RUg46njPa9Wp4azjz1DVj0K+KOYD8T0jGE7IhoqaaRchxNUTz5O7GYFkLObjN4xEJKxVPSLyBxx3D8AvcFb6xC9hbuCqvqIH2oirJ8OycBlGIhOOxf9LIBf4G85mrE5uWfwSRjz+koqaSVigjezcDSh+i9djGAlJgxa/qu4DboiCLNEjXIu/FsXfFnbuSnIykpJiyzmNiKKqOPs4jUijjVzuGU48/vdFpF3IdXsRebfxosUQYe7crc3V0xY2cAGWftGIKKmpqezZsyeuo9jGC6rKnj17SE1NDbtNOD7+Tqq6P+Qh+0SkcxPkix3C3MBVm8Xv8/jweXxxHbIBLEKnEVl69OjB5s2bafHIukatpKamBncbh0M4it8vIr1UdSOAiPSmjoiacUMYq3oOVNbMtxsg1Zsa164esCxcRmRJSkqib9++rS2GUQfhKP47gQUi8jFOCIafAJMjKlWk8XhBmza5C467J+5dPRkZpvgNI0EJZ3L3nyJyLE76RMWJq7874pJFkgZcPVX+KkqrSmtdxw/OBG/8u3oyqdi+vbXFMAyjFahzctcNx5wD4Cr6YuB04EoRSY6SfJGhAcVfV2TOAG3C4rdkLIaRsNS3quc13B26IjIcmAVsBIYBf4m4ZJGkAR9/XQHaAqR4U+Lex+/JzLDlnIaRoNTn6klT1a3u+S+A51T1YRHxAEsiLlkkkfqTrdeVfStAqi81rjdwgaVfNIxEpj6LP3TnxSnAPADVBnIWxgMNuHrqyr4VoE24ejIy0fJy/OXxmzTeMIymUZ/F/6GIvAZsA9oDH0IwT258a4uGFH9DFr83lX2l+yIiWrQIxuspLsaTHN9TNoZhNI76LP4bceLzrAdOUNUKt7wrzhLP+CWQbL2OXYUNWvy+tuDjt5j8hpGo1Gnxq7PXemYt5d+E07GIPAecA+xU1aPdsinA1UBgO98dqvpOI2VuPh532P4q8NZ8CxpS/Kne1Ph39WRahE7DSFTCic7ZVGYAZ9ZS/qiqDneP6Ct9cDZwQZ3ungMV9S/nbAvr+C39omEkLhFT/Kr6CbA3Uv03iwYUf0JM7rqKv8qWdBpGwtEoxe9G5hzazGdeJyLLROQ5EWlfz7Mmi8hiEVnc4oGegq6eOhR/A5O7gXX88Rx50JMRsPhN8RtGohFOWOb5IpItIh2Ar4HpIvJIE5/3JNAfGI6zWujhum5U1adVNU9V83Jzc5v4uDoI9fHXQkllCT7xkeRJqrU+1ZeKX/1UNhDhM5YxH79hJC7hWPw5qloAXAi8qKqjgVOb8jBV3aGqVe5egOnAqKb002wacvVUlJCWlFZnEom2EJPfsnAZRuISjuL3uWv3LwXeas7D3H4CXACsaE5/TSZg8dcRobOu7FsB2kLeXUlPBxFLxmIYCUg4YZnvBd4FFqjqIhHpB6xpqJGIvAKMAzqJyGbgP4Fxbtwfxdkf8B9NE7uZBBR/VUWt1SUVtWffCpDii/+8uyLiBmozH79hJBrhhGWehROgLXC9DrgojHYTail+tlHSRYrs7s7r3rXQvneN6gOVB8Ky+ONZ8YNF6DSMRCWcyd0H3cndJBGZJyK7ROQX0RAuYnQ/FhDYvLjW6rry7QYI+Pjj2dUD4LUInYaRkITj4z/dndw9B8c9czhwSySFijipOZA7EDYvqrW6vuxbEOLqiePJXXCWdJrFbxiJR1iTu+7r2cAsVc2PoDzRo0eeY/HXshY/XFdPvO/e9WRkUGWregwj4QhH8b8lIquBEcA8EckF4tvUBegxEg7shb3ralQ1NLmb6nN9/PFu8dvkrmEkJA0qflW9HTgeyHMjdJYA50VasIjTI895rcXPX1JZUmf2LWgbyznBzcJlrh7DSDjCmdxNB36Ds+sW4DAgL5JCRYXcgZCcWcPPr6oNT+62geWcYFm4DCNRCcfV8zxO4pXj3estwNSISRQtPF5ndU81xV9WVYZf/fVP7raRVT2ejEz8xcWoP/6TqhmGET7hKP7+qvogUAGgqiUcmpYxfukxEnasADcMM4RE5qzPx99mXD1uoLaSAw3caRhGWyIcxV8uImk4u20Rkf5AfGu8AN3znHg925YGi4KROcNZzhnnrp5goDZb2WMYCUU4iv8/gX8CPUXkZZyk67dGVKpoEZzgPejuCcfiT/Ik4RVv/Fv8GRah0zASkXBCNrwvIl8Dx+G4eH6rqrsjLlk0yOwM7XofqvjDsPihbSRjsSxchpGYhJuIJRXYBxQAg0TkxMiJFGV6jDxkSWc4Fj84a/nj39XjZuEyxW8YCUWDFr+ITAP+DVgJBJZ/KPBJBOWKHj3yYMVsKNgK2YcdzLcbhsUf966eTMvCZRiJSDhhmc8HjlTV+NZyddFjpPO6eTEMGh+0+OvbwAWuqyfeLf4Mc/UYRiISjqtnHVB7DsK2QNch4E0O+vkbyrcbIM2XFvcWvze4qscsfsNIJMKx+EuAJSIyj5BlnKp6Q8Skiia+FOg2LOjnD/r4E2By12PpFw0jIQlH8b/pHqHUDGkZz3TPg69mQFVlUPEHArHVRYovJe6jc0pSEpKSYpO7hpFghKP426nq/4QWiMhvIyRP69AjD758EnaudBKt+9LwSP1esFRvKgVlBVESMHJYhE7DSDzC8fFPrKVsUkONROQ5EdkpIitCyjqIyPsissZ9bd8IWVuMsqoyPt/6ORqIxR+c4F3UYKL1AG3B1QMWodMwEpE6Fb+ITBCRvwN9ReTNkOMjYG8Yfc8AzqxWdjswT1WPwNkBfHsT5W4WTy19isnvT+ajTR85Be16QUZn2Ly4wVj8AVJ9qXHv6gHwWhYuw0g46nP1/AvYBnQCHg4pLwSWNdSxqn4iIn2qFZ8HjHPPXwDmA7eFJ2rLUFheyMzVMwF49KtH+UmPn5DkSQpm5CrJGZNgFn+mZeEyjASjTotfVTeo6nxVHaOqH4ccX6tqZROf10VVt7nn24Eudd0oIpNFZLGILN61a1cTH1eTV797lcKKQn497NesL1jP377/m1PRIw/2rOFAWUFYFn9b2MAFro+/uKS1xTAMI4rU5+pZ4L4WikhByFEoIs2e1VTHwV7n6iBVfVpV81Q1Lzc3t7mPA5xcui+teomx3cfy62G/Jq9LHn9Z+heKyouCfv6Skt1hWfxtxdXjyTAfv2EkGvVN7l4OoKpZqpodcmSpanYTn7dDRLoBuK87m9hPk/jbmr+xt3QvVw+5GhHh5ryb2Vu6l+dWPAeHHQPioaQsP2yLv1IrqfBXREHyyGGTu4aReNSn+N8InIjI6y30vDc5uEpoIjC3hfptkIqqCp5f8TzHdj6WEV1GADC402B+1vdnvLjqRbZXFkPuUZRUFDcYrgFCkrHEudVv6RcNI/GoT/GHZtnq19iOReQV4HPgSBHZLCK/Ah4AThORNcCp7nVU+Pu6v7OjZAdXD736kPIbjr0BVeWJb56AHnmU+MtJD0fxuxu84n2C15ORiVZU4C8vb21RDMOIEvWt6tE6zsNCVSfUUfXTxvbVXKr8VTy7/FmO6nAUYw8be0hd98zuXD7ocmasmMEVh19JyV4hfc86WP12tV4Eeh8Pae2ANpR3NyQmv6dDh1aWxjCMaFCf4h/mTuIKkBYyoSs4c7NN9fNHnfc2vMfGwo08Mu4RRGqmC75qyFW8seYNpu1dTIUI6Wveh0W1eLdGTYaf/Tdw0OKPd1dPMP1iURGY4jeMhKBOxa+q3mgKEilUlenLp9Mvpx8/7VX7j43s5GyuGXYNDyx0PE/px10HvavtPfvHbbDhX8HLgMVfl6unoLyALYVbOKrjUS0wishhWbgMI/EINwNX3PLx5o9Zs28NVw25qt74O5cOuJReWb0ASO9wuBOxM/ToexLsWAmlzg+f4ORuHa6e55Y/x5X/uDLmV/0cjNBp8XoMI1Fo04pfVZm+bDrdM7tzZt/q0SMOJcmbxI0jbgSgXUq7mjf0HAVoMG5/is+1+OtIxrJ2/1pKq0rZUrilqeJHBUu/aBiJRzjROeOWl5bMY9nuZZzZ9VpmL97W4P2qA7iy93+zbXtfXtmx8ZC6pIpuXCQeZNNCOPynDVr8Gwud9usL1tMnp0/zBhJBDmbhMovfMBKFNq34Z377Bv6KLGbN78osXd6Ilt/WWnpC+/503fQFUL+Pv8pfxabCTQBsKNjQOKGjTHBy1+L1GEbC0KYV/0vnPcS6fRvomdWn2X1d/eJilpceSdfN86GqMujqqW1Vz/aS7UHf/vqC9c1+diSxyV3DSDzatOLvmJFGx4yBLdLXEV0yWfBdf06regt2riI1pxtQu6snYOX7PL6Yt/glLQ08HvPxG0YC0aYnd1uS/rmZzCvp61xs+rLeyd1NBY6bJ69LHhvyY1zxi7iB2szHbxiJgin+MOmfm8FmzaUivTNs+pI0rxPWoTYf/4bCDaR6UxnZdSQ7D+ykuCK2laoTmjm2ZTQMo+UwxR8m/XIzAWFXu+Gw8Ut8Hh8e8dRq8W8s2EjP7J70zXF+IcS6u8drEToNI6EwxR8mvTum4xH4Pnkw5G9ECrfVmYxlY+FGemX1ok92HyD2Fb8nI9NW9RhGAmGKP0xSfF56dkhnYdURTsGmL0n1ptZQ/FX+KjYXbqZXdi96ZvVEkJhf2ePJzKTKfPyGkTCY4m8E/Tpl8GlhN/ClwkZngre6q2db8TYq/BX0zupNqi+VbhndYt/it5j8hpFQmOJvBP1zM1mzpww97Ng6Lf7Ajt1e2U7cn97ZvVmfvz7aojYKy8JlGImFKf5G0C83k9IKP4Wdj4Xty0jxJNVY1bOxwFX8WQcV/4aCDTgphmMTr+XdNYyEwhR/I+if64Q32JgxFPyVpFRV1ti5u6FgA2m+NDqndwagT04fiiqK2FO6J+ryhosnIxN/SQnq97e2KIZhRAFT/I3AWdIJyxgAQGrFgRqunk2Fm5xJXTfhSzys7Alm4SopaWVJDMOIBq2i+EVkvYgsF5ElIrK4NWRoCp0yk8lK9fFtfhJ0GkBKWWENV8+Ggg1BNw84rp5AeaxySBYuwzDaPK1p8Z+sqsNVNa8VZWgUIkL/3EzW7S6CnqNJLdlPWciqnkp/JZuLNgcndgG6ZXQjyZMU00s6LVCbYSQW5uppJP1yM1i7sxh6jialqpzS8oPKclvxNir9lUErH8Dr8dIrq1dMr+zxmOI3jISitRS/Au+JyFciMrmVZGgS/XMz2V5QSknXPFL9fkorDirLQHC2nlk9D2nTJ6dPbLt63PSLtonLMBKD1lL8J6jqscBZwLUicmL1G0RksogsFpHFu3btir6EdRBY2bPO341Ub8ohq3o2FDrKPdTiD1xvLNxIlb8qeoI2ArP4DSOxaBXFr6pb3NedwBvAqFrueVpV81Q1Lzc3N9oi1klgZc/a3cWkZB1GqR5U5hsLNpLmSyM37VB5+2T3odJfydbirVGVNVyC6RctQqdhJARRV/wikiEiWYFz4HRgRbTlaCqBYG1rdxWTktOTSoGqoh2As2s3dClnsE2Mr+zxuqt6KnfvbmVJDMOIBq1h8XcBFojIUmAh8Laq/rMV5GgSgWBta3cVkdreCbtctuEzwLH4q7t54KDij9UJXk9ODqlDh7L3ueeo2LmztcUxDCPCRF3xq+o6VR3mHoNV9Y/RlqG59OuUwbpdxaS07wdA6Q/vU7l9JZsLN9HLmwE7VztHsWNBd0jtQFZyVswu6RQRDnvgAfxlZWy7666YDi9hGEbzseWcTaB/biY/7i4iOcnxjZcte5Vtz/yESq2i15fPwl9GO8cTx0LJXkSEPtmxvbInpV9fOt98M8WffMr+V19tbXEMw4ggpvibQCBY24Fy5+0rPesBNp58OwC9xt4MFz8P5zwKpfnw5VPAwWBtsUz7n08gY+xYdkx7kPL161tbHMMwIoQp/iYQWNK5r8hxiZT1O5ENHXoA0HvYFXD0hZD3SzjybPjyr1BaQO/s3mwr3lZrqsZYQTweut3/RyQ5mS233YZWVra2SIZhRABT/E0gsKRzd6ETzbK0spSNhc5Szk5pnQ7eeOLvoHQ/LH42LoK1ASR16ULXu/9A6dJl7Jk+vbXFMQwjApjibwKBYG27ChzFX1ZVxsYCJ8/uIUs5u4+A/qfA53+mT0Y3IPYVP0DO2WeTffbZ7PrzXziwYmVri2MYRgtjir8JBIK1bc93Nm+VVZU5CdZDgrMF+cnNULyLXms/ARqv+FWVWz+5lce/fhy/Ri9efte7/4CvY0e23nor/tLYdU8ZhtF4TPE3kX65GWzZ4yj+ovIithRuqXUNP33GQq/jSf/8STqndW70ks5Pt3zKP378B9OXT2fKv6ZELeyDNyeHbvf/kfJ169h0za8p/uILW+ZpGG0EU/xNpH9uJrtcH/+PBT9SqZWHxOE/hBN/BwVb6ONJaZTFr6o8vexpumV0Y/LQybzxwxvc9ultVFRVtMQQGiRz7Fi63HknZatXs3HSv/Pj+PHsm/mqJWwxjDjHFH8T6Z+bAZoEwJp9awBqd/UA9P8pdBtO772bGrV7d9H2RSzdtZRfHv1Lrj/mem7Ou5l317/LTfNvqpH5K1J0uOIXHD7/I7r98Y+QlMT2KVNYM+5kdkx7kIotW6Iig2EYLYsp/ibSLzcT9fsA+H7f90DNqJxBRODEW+hdtI/88nz2l+4P6xlPL3uaTmmduOCICwCYOHgifzjuD3yy+ROu/eBaSiqiY3l7UlNpd9GF9H39dXq//L9kjD2evS++yLoLLqRs7dqoyGAYRsthir+J9O6YjgfH4t9UuIl0XzodUzvW3eDIn9E3vSsA6/N/bLD/JTuX8OX2L5k0eBIp3pRg+aVHXsofT/gji3cs5ur3rya/LL95A2kEIkL6iBH0ePRR+r/9FpKczKarJ1MZQ2GzDcNoGFP8TSTF56VH+2zAWb7ZK7tXjaich+Dx0HvE1QBs+G5ug/1PXz6ddintuGTAJTXqzu1/Lg+f9DDf7vmWif+YyLr965o0huaQ3KcPPZ98ksp9+9j0H9dYSGfDiCNM8TeD/p0yEXXcPXVO7IZw2DET8SlsWPM21LNC5ts93/LJ5k+4YtAVpCel13rPT3v/lCdPfZJ9Zfu47O3L+PvavzdtEM0gbcjRdH/kYUpXr2bL//ud7fQ1jDjBFH8z6J+bid/vuHvqnNgNIcmXSo+UDqwv2wuzJjqxfGph+vLpZCVlMWHghHr7G91tNLPOncWgjoO4Y8Ed3P3Z3RyoPND4gTSDrJNPpuvdf6Do44/ZPnWqLfk0jDjAFH8zCJ3gDcfiB+jd+WjWt+8O374FT50EW5ccUr92/1re3/A+E46aQFZyVoP9dU7vzDOnP8PkoZOZ88Mcfv72z1m7P7oTru0vu4yOV1/F/pmvsueZZ1q8f1WlqqCAsrVrKf78c/LffJOizz5r8ecYRqLga20B4pnQJZ11ruipRp/svnyx7Uv8k97CM/tX8OxpcMb9MPIqEOGZ5c+Q5kvjF0f9Imw5fB4f1x9zPSM6j+D3C37PhLcncNdxdzG+//gmjasp5N50ExVbtrLr4UdIOuwwcs4+u9F9+MvLKV+7ltLvvqNs9XeUff8d5Zs2U7lrF1rL7uEOkybR+ZabEa+3JYZgGAmDKf5m4Fj84bt6AHrn9Kasqow/7VnMyZdOZ9D8R/C+czNs+BebTr6Fd358hyuOuoL2qe0bLc/x3Y9n1rmzuO2T27hzwZ0s27WM20beRpI3qdF9NRbxeOj2wH9RuWsXW2+7ndKVq+j0m98E0zrWRfmmTex55lkOfP01ZT/+CO48gaSkkHLEEaQNHYqvc2fnyM0NHvv+7//YO2MG5evXc9hDDzX4HMMwDiLx4JPNy8vTxYsXt7YYNVBVhj59Dt7UnSy8/Iv6V/W4bC3awq2f3MKqvU7ws+zkbEYndWDMhm9YmN2BD5PhnezjyPWkNNBT3VSqnycqtzKj+AeOyT2GR05+5NCooRGkqrCQHdOmkT/7dXydO9P51lvJPvtnNd6bip072fPXv7Jv1mzE4yHjuONIGTiQ1CMHkDJwIMm9ezdoye975RW2T/0jKf370/PJv5DUvXskh2YY8UitSqlVFL+InAn8D+AFnlHVB+q7P1YVP8Co5y6hqLyQkvXXN6qdeIvwZvyAL2MN3ow1eJIKABifX871e5q/K7c9RXyYmcTduR3J9qbxyMjfM+zIC5zNZFHgwNKlbL/3PkpXriR95Ei6/OEuUgcMoCo/nz3PPMPel/4Xrayk3cUX0enXvyGpS+cmPafos8/YcuNNSHIyPZ54gvRjj2nhkRhGXBMbil9EvMD3wGnAZmARMEFVV9XVJpYV/9xVi/l6425yk/s3uQ9VZX/lZnaUfUff9DGkeJrvtti0bQf+b99mWMrHvNxtH9t9Xu4oS3b2BXQ+CnJ6Qk4PyMgFT/hz/KpKhb+CsqoyBCE9KR2P1N5eq6rYP2s2ux59lKqiIrJOP43iBZ/hLyoi+5xzyL3+OpJ7heciq4+ydevY9OtfU7l1G51/fztJhx2GPz+fqv37qcrPp2p/Pv7iIpJ69yZt6DDShg7Bm53d7OcaRhwQM4p/DDBFVc9wr38PoKr/VVebWFb8scyuwjJeXbSRN7/4kqp209mZsZdzC4vpV1FBvsdDvtfDPq+Pfb4U8r0+KkXwAxo8FAUqgXJRKlDKUTTko+RRSMdDhnrJxOu8io/2nkzaJ+fQMbUTnbUdvd5bS8ony/DmDSV54sX4Du+LShLi9YEnCY/XS5InCZ/Hi8/rxev1kuzx4fV43edJUC7k4Llf/fhRKvbuZfctd1C++JtD3wSPB7IyITUZdu4J7p/w9elN2tChpA8bhq9jiBss5BeReD1IUpJzJCcHz/H5EF8SkuRz3FE+n1Pn9YLHi3jEea7H47i4PB6nX7fvcFyChtFCxIzivxg4U1Wvcq+vAEar6nV1tTHF3zwqqvy8u3IrDy96nN2+fwAgfi8pVT7S/UJOldLeX0GyKgKHHuqsAEjyQ5JCsjqvSeoo3lKPUuJVSjxKiQeKPVDkhX1eKPJW+yWgGlFXk7dKGbAFyn1QlApFaVCSCuo+M61U6b9dOWIrHL5VOWKr0q4VNxz7IfhvqRD8QtVqb5GG3ENIm+o09j+5+nOM2GTLr87g3N8+1tTmtf6VY3ZVj4hMBia7l0Ui8l0DTToBuyMrVUxi4w5haQONFkVGlmhif+9E48bVnbjxf5o69n+q6pnVC1tD8W8BeoZc93DLDkFVnwaeDrdTEVmsqnnNFy++sHEnFjbuxCMSY2+NnbuLgCNEpK+IJAOXAW+2ghyGYRgJSdQtflWtFJHrgHdxlnM+p6qW0dswDCNKtIqPX1XfAd5p4W7Ddgu1MWzciYWNO/Fo8bHHxc5dwzAMo+Ww6JyGYRgJRtwrfhE5U0S+E5EfROT21pYnkojIcyKyU0RWhJR1EJH3RWSN+9r46G4xjoj0FJGPRGSViKwUkd+65W167CKSKiILRWSpO+573PK+IvKl+5l/1V0k0eYQEa+IfCMib7nXbX7cIrJeRJaLyBIRWeyWtfjnPK4Vvxv+4c/AWcAgYIKIDGpdqSLKDKD6mtzbgXmqegQwz71ua1QCv1PVQcBxwLXu37mtj70MOEVVhwHDgTNF5DhgGvCoqh4O7AN+1XoiRpTfAt+GXCfKuE9W1eEhSzhb/HMe14ofGAX8oKrrVLUcmAmc18oyRQxV/QTYW634POAF9/wF4PxoyhQNVHWbqn7tnhfiKIPutPGxq0ORe5nkHgqcAsx2y9vcuAFEpAdwNvCMey0kwLjroMU/5/Gu+LsDm0KuN7tliUQXVd3mnm8HurSmMJFGRPoAxwBfkgBjd90dS4CdwPvAWmC/qgYSHLfVz/xjwK24kS2AjiTGuBV4T0S+cqMXQAQ+5zEbssFoPKqqItJml2mJSCbwOnCjqhaEBjtrq2NX1SpguIi0A94ABrauRJFHRM4BdqrqVyIyrpXFiTYnqOoWEekMvC8iq0MrW+pzHu8Wf1jhH9o4O0SkG4D7urOV5YkIIpKEo/RfVtW/ucUJMXYAVd0PfASMAdqJSMBoa4uf+bHAeBFZj+O+PQUnf0dbHzequsV93YnzRT+KCHzO413xW/gHZ7wT3fOJwNxWlCUiuP7dZ4FvVfWRkKo2PXYRyXUtfUQkDSeHxbc4XwAXu7e1uXGr6u9VtYeq9sH5n/5QVS+njY9bRDJEJCtwDpwOrCACn/O438AlIj/D8QcGwj/8sXUlihwi8gowDidS4Q7gP4E5wGtAL2ADcKmqVp8AjmtE5ATgU2A5B32+d+D4+dvs2EVkKM5knhfHSHtNVe8VkX44lnAH4BvgF6ra/LRtMYjr6rlZVc9p6+N2x/eGe+kD/k9V/ygiHWnhz3ncK37DMAyjccS7q8cwDMNoJKb4DcMwEgxT/IZhGAmGKX7DMIwEwxS/YRhGgmGK34gYIqIi8nDI9c0iMqWF+p4hIhc3fGezn3OJiHwrIh/VUneEiLwlImvdLfYficiJkZapLkTk/NAghSJyr4ic2lryGLGLKX4jkpQBF4pIp9YWJJSQ3Z/h8CvgalU9uVofqcDbwNOq2l9VRwDXA/1aTtKauBFp6+J8nCi1AKjq3ar6QSTlMeITU/xGJKnESRt3U/WK6ha7iBS5r+NE5GMRmSsi60TkARG53I1Lv1xE+od0c6qILBaR7934LoGgZv8tIotEZJmI/EdIv5+KyJvAqlrkmeD2v0JEprlldwMnAM+KyH9Xa3I58LmqBneKq+oKVZ3hts0QJ3/CQjem/Hlu+SQR+ZuI/NONr/5giAyni8jnIvK1iMxyYxMFYrRPE5GvgUtE5Gp3fEtF5HURSReR44HxwH+LE8u9f+h7LCI/deVY7sqVEtL3Pe4zl4vIQLf8JLefJW67rIb+2EYcoap22BGRAygCsoH1QA5wMzDFrZsBXBx6r/s6DtgPdANScOKx3OPW/RZ4LKT9P3GMlyNwojWmApOBu9x7UoDFQF+332Kgby1yHgZsBHJxdkx+CJzv1s0H8mpp8wjw23rGfj/OzlKAdsD3QAYwCVjnvh+pODsxe+Lsxv4EyHDb3Abc7Z6vB24N6btjyPlU4Po63tMZOCEOUnGi2A5wy1/ECXQX6DvQ/jfAM+7534Gx7nkm4Gvtz5MdLXeYxW9EFFUtwFE0NzSi2SJ1YvCX4YQhfs8tXw70CbnvNVX1q+oaHGU6ECe+yZXihDL+Eiec7xHu/QtV9cdanjcSmK+qu9QJ+/sy0ChfvYi84f5aCASQOx243ZVjPo7y7eXWzVPVfFUtxfn10Rsnwcwg4DO3zUS3PMCrIedHu79eluP88hjcgHhHAj+q6vfu9QvVxheQ+SsOvr+fAY+IyA1AOz0YDtloA1hYZiMaPAZ8DTwfUlaJ62oUEQ8QmkYvNP6KP+Taz6Gf2erxRhQQHAv23dAKN+ZLcVOEr4OVhChPVb1ARPKAhwKPBC5S1e+qyTGaQ8dXhTMmAd5X1Ql1PC9U9hk4v0iWisgknF8zzSEgT0AWVPUBEXkb+BnOl9EZqrq6rg6M+MIsfiPiqBNQ6jUOTZW3Hhjhno/HyS7VWC4REY/r9+8HfAe8C/xanDDOiMgAcSId1sdC4CQR6eROnk4APm6gzf8BY0VkfEhZesj5u8D1Ik7SABE5poH+vnD7O9y9P0NEBtRxbxawzR3j5SHlhW5ddb4D+gT6Bq6ggfGJSH9VXa6q03Ci4Lb5PACJhCl+I1o8jOPHDjAdR9kuxYkx3xRrfCOO0v4HcI3rOnkGx33ytThJ6Z+igV+26mQ3uh0n7O9S4CtVrTf0raoeAM4BrnEnoT8H7sLxuQPch/NltkxEVrrX9fW3C8f//4qILAM+p25l+wccN9ZnQKgVPhO4xZ2MDU6Cu+/LvwOzXPeQH/hrffIAN7quq2VABc57bLQRLDqnYRhGgmEWv2EYRoJhit8wDCPBMMVvGIaRYJjiNwzDSDBM8RuGYSQYpvgNwzASDFP8hmEYCYYpfsMwjATj/wMFIy5tu8GhdAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], "source": [ "state = strategy.initialize(rng, es_params)\n", "for i in range(num_gens):\n", @@ -159,7 +209,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "c8bb5d1f-2b3b-4c3a-91fd-58c4be4b10ce", "metadata": {}, "outputs": [], @@ -190,10 +240,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "772c9449-6fab-4650-bd82-d0c589eb45b1", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ParameterReshaper: 4610 parameters detected for optimization.\n" + ] + }, + { + "data": { + "text/plain": [ + "4610" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from evosax.utils import ParameterReshaper\n", "\n", @@ -212,10 +280,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "40ff50bd", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(100, 4610)" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from evosax import DE\n", "strategy = DE(popsize=100, num_dims=param_reshaper.total_params)\n", @@ -234,34 +313,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "607fab0a", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "(frozen_dict_keys(['params']), (100, 4, 64))" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "net_params = param_reshaper.reshape(x)\n", "net_params.keys(), net_params['params']['Dense_0']['kernel'].shape" ] }, - { - "cell_type": "markdown", - "id": "38cb1644", - "metadata": {}, - "source": [ - "If you now want to map over the population member axis, you can do so with the of the `vmap_dict` (more about this later):" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4b1ac3ea", - "metadata": {}, - "outputs": [], - "source": [ - "# Get dictionary to vectorize/parallelize rollouts with\n", - "param_reshaper.vmap_dict" - ] - }, { "cell_type": "markdown", "id": "4d1192e9", @@ -274,10 +345,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "9c488bab", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "DeviceArray([ 0.49, -0.04, -0.59], dtype=float32)" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from evosax import FitnessShaper\n", "fit_shaper = FitnessShaper(centered_rank=True, w_decay=0.01, maximize=True)\n", @@ -294,28 +376,39 @@ "source": [ "## ARS on CartPole Task\n", "\n", - "`evosax` also comes with a simple fitness evaluation helper for a JAX-based version of Cartpole. You will have to make use of the `vmap_dict` in order to vectorize the rollouts along the population axis:" + "`evosax` also comes with a simple fitness evaluation helper for all [`gymnax`](https://github.com/RobertTLange/gymnax) environments (e.g. CartPole, MinAtar, etc.). We will vectorize the rollouts of the different population members:" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "92cbdc9d-fe18-4582-91c8-4713568c1199", "metadata": {}, "outputs": [], "source": [ - "from evosax.problems import GymFitness\n", + "from evosax.problems import GymnaxFitness\n", "\n", - "evaluator = GymFitness(\"CartPole-v1\", num_env_steps=200, num_rollouts=16)\n", - "evaluator.set_apply_fn(param_reshaper.vmap_dict, network.apply)" + "evaluator = GymnaxFitness(\"CartPole-v1\", num_env_steps=200, num_rollouts=16)\n", + "evaluator.set_apply_fn(network.apply)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "id": "5186a497", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "EvoParams(opt_params=OptParams(lrate_init=0.05, lrate_decay=1.0, lrate_limit=0.001, momentum=0.0, beta_1=None, beta_2=None, beta_3=None, eps=None, max_speed=None), sigma_init=0.03, sigma_decay=1.0, sigma_limit=0.01, init_min=0.0, init_max=0.0, clip_min=-3.4028235e+38, clip_max=3.4028235e+38)" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "from evosax import ARS\n", "\n", @@ -330,15 +423,66 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "c99235f5-6cb1-4e3b-b00b-1b5789d7898e", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/rob/anaconda3/envs/mle-toolbox/lib/python3.9/site-packages/flax/core/scope.py:740: FutureWarning: jax.tree_leaves is deprecated, and will be removed in a future release. Use jax.tree_util.tree_leaves instead.\n", + " abs_value_flat = jax.tree_leaves(abs_value)\n", + "/Users/rob/anaconda3/envs/mle-toolbox/lib/python3.9/site-packages/flax/core/scope.py:741: FutureWarning: jax.tree_leaves is deprecated, and will be removed in a future release. Use jax.tree_util.tree_leaves instead.\n", + " value_flat = jax.tree_leaves(value)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation: 0 Performance: 22.875\n", + "Generation: 5 Performance: 26.25\n", + "Generation: 10 Performance: 27.8125\n", + "Generation: 15 Performance: 31.3125\n", + "Generation: 20 Performance: 53.0\n", + "Generation: 25 Performance: 99.0625\n", + "Generation: 30 Performance: 115.8125\n", + "Generation: 35 Performance: 130.125\n", + "Generation: 40 Performance: 192.9375\n", + "Generation: 45 Performance: 200.0\n" + ] + }, + { + "data": { + "text/plain": [ + "(
,\n", + " )" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAADgCAYAAADsbXoVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAB3DElEQVR4nO2dd3hb5fmw70fT8t524kxnEggJJAGSsDeFsmdZgbIplFHo/BUKbT+g0FJaSoECYYe9QimUVQgrJIwMsryS2In3trX1fn+cI0W2JVtO7NgJ731duqyz3vMeST7PebYopdBoNBqNBsAy1BPQaDQazfBBCwWNRqPRRNBCQaPRaDQRtFDQaDQaTQQtFDQajUYTQQsFjUaj0UTQQkGz3YjIoSJSOdTz2NURESUiE4fBPBaIyJKhnsfOQkRuFZGnhnoeww0tFHYBRORHIrJMRNpFZKuIvCUiB+7AeF1uQubNPWSO3yYi60TkooGZfZ9zudWcz/4743wDzWDfSEXkQxHxmN9NvYi8LCIjBut8OwsR+bGIrDV/bzUi8m8RSRvqeWm0UBj2iMgNwL3AH4ECYAzwD+Ck7RjL1svmLUqpVCAd+DnwsIhM6/eE+zcfAS4AGs2/mtj8xPxuJgKpwN1DPJ8dQkQOwfg9n6OUSgP2AJ4bhPOIiOh7XD/RH9gwRkQygNuAq5VSLyulOpRSfqXUG0qpm8x99hORz0Sk2dQi/i4ijqgxlIhcLSIbgA0i8pG56Vvz6fOs6HMqg1eBJmCaiDhF5F4R2WK+7hURZ5z5jhSRl0SkTkTKReTaPi7xIGAEcC1wdrd5d1HtRWSceS02c3m8iHxkPmm+KyL3h/eP2vciEdksIk0icoWIzBGRFeZn9fduc79YRNaY+74tImO7fYZXiMgG89j7zRvOHsA/gbnmZ9ls7u8UkbtFZJP5FPxPEXFFjXeT+V1tEZGL+/iMIiilmoFXgZlRY11kzrtNRMpE5PKobYeKSKWI3CgiteY5L4raniMir4tIq4gsBSZ0+0zmiciXItJi/p0Xte1DEfm9iHxqXvsb5nhPm+N9KSLj4lzKHOAzpdTX5nU1KqUeV0q19fX5iUiWiCw2f2NN5vtR3eb1BxH5BOgEikVkTxH5r4g0muP9KmouDhF5wvz8VovI7ES/j90WpZR+DdMXcCwQAGy97DMLOACwAeOANcB1UdsV8F8gG3BFrZsYtc+hQKX53gKcAviBKRhC6XMgH8gDPgVuj3PccuC3gAMoBsqAY3qZ+yPA84AdaABOi9p2K/BU1PI4c942c/kzjCdmB3Ag0BreP2rffwJJwNGAB+OGmg8UAbXAIeb+JwElGE+sNuA3wKfdPsPFQCaGplYHHGtuWwAs6XZdfwFeNz/zNOAN4P9Ffac1wF5ACvBM9++j21gfApeY73OAd4HXorYfj3EzF+AQjBvhvlHfT8D8Du3AD8ztWeb2Rebnn2LOpyp8Lebcm4Dzzc/kHHM5J2peJea5M4DvgPXAkeb+TwCPxbmmgwA38DtgPuDsx+eXA5wGJJvbXgBe7fZ5bQL2NOeRBmwFbjR/C2nA/lG/MY/5uViB/wd8PtT/90P9GvIJ6FcvXw6cC1T385jrgFeilhVweLd9YgmFENCMYcr5Bjjb3FYK/CBq32OAiqjjwkJhf2BTt/P8spcbQzLGjfxkc/lBut7sbiWOUMC4MQeA5KjtT9FTKBRFbW8AzopafglTeAJvAT+O2mbBuHmOjfq8Doza/jzwC/P9AqKEAsbNuQOYELVuLlBuvn8UuCNq2+Tu30e3z+lDcy4t5n7fAGN6+f5fBX4a9f24iXqowBCGB5g3QT8wNWrbH9kmFM4HlnYb+zNgQdS8fh217R7grajlHwLf9DLP4zBu9s1AO/Bnc069fn4xxpkJNHX7vG6LWj4H+DrOsbcC70YtTwPc2/v/uru8erMxa4aeBiBXRGxKqUCsHURkMsY/1GyMG60N44k9ms0JnGuLUmpUjPUjgY1RyxvNdd0ZC4wMm1BMrMDHcc53CsaN/d/m8tPAuyKSp5Sq62OuI4FGpVRn1LrNwOhu+9VEvXfHWE6NmvtfReSeqO2CoVGEr706altn1LHdycP4HpaLSPRY1qi5R38/0Z9tPK5VSv1LRKZjaCyjMJ6GEZHjgFswhIvFPPfKqGMbuv12wnPPw/itRP82oufS/XsPby+KWk708+2BUuot4C0xbP6HYTzxrwNeoZfPT0SSMTSJY4Esc3uaiFiVUkFzOfqaRmM82MSj+/ea1Nv/2/cB7VMY3nwGeIGTe9nnAWAtMEkplQ78CuOfKJodKYW7BeOmGWaMua47mzGe5jKjXmlKqR/EGfdCjJvGJhGpxrgp2IEfmds7MG4OYQqj3m8Fss0bRJjuAqE/bAYu7zZ3l1Lq0wSO7f7Z1mPcEPeMGitDGY7i8Nyj5zom0UkqpVYCvwfCPg0nhsZzN1CglMrEELLdv/9Y1GEI5Xhz6f69h7dXJTrfRFBKhZRS7wHvY5iw+vr8bsQwa+5v/t4PNtdHX3P0d7IZw5SpSRAtFIYxSqkWDBv9/SJysogki4hdRI4TkbvM3dIwzDDtIjIVuDKBoWtI/B/lWeA3IpInIrnmfGLFdi8F2kTk5yLiEhGriOwlInO67ygiRcARwAkY6v9MYAZwJ9uikL4BDhaRMWI43H8ZPl4ptRFYBtwqIg4RmYthrthe/gn8UkT2NOeXISJnJHhsDTBKTCe5UioEPAz8RUTyzfGKROQYc//ngQUiMs0Uarf0c66PY0ShnYjhT3Fi3uBNreHoRAYxn6pfxvgMk8WINLswapd/A5PFCIe2iRGQMA1DU9khROQkETnbdBqLiOyH4Q/5PIHPLw1DaDSLSDZ9f36LgREicp3pwE6TXTT8eWehhcIwRyl1D3ADhvOzDuPJ5ycYtmOAn2E8Xbdh/DMlEtp3K/C4GJE0Z/ax7+8xbsArMMwSX5nrus8zyLabfDnGE9+/MJyQ3Tkfw978jlKqOvwC7gP2FpG9lFL/Na9lBYa5pfvN6FwMW3ODOZ/nMLSqfqOUegVDIC0SkVZgFYbNOxHeB1YD1SJSb677OYYT9nNzvHcxnm7DZpN7zeNKzL/9masP+Cvwf8qI1rkWQ9A0YfwOXu/HcD/B0NaqgYXAY1HnacD4Pm/E+IxvBk5QStX3HKbfNAGXAhswAwSAPymlnja3x/38MD47F8bv63PgP72dyPyMjsJ4aKg2z3nYAFzDbouYDhaNZpdGRJ4D1iql+vvkrdFootCagmaXRIycgwkiYhGRYzHCSl8d4mlpNLs8OvpIs6tSiGETzwEqgSuVmQyl0Wi2H20+0mg0Gk0EbT7SaDQaTQQtFDQajUYTYZf2KRx77LHqP//pNSJNo9FoND2Jm+C4S2sK9fUDETKt0Wg0mjC7tFDQaDQazcCihYJGo9FoIgyaUBCR0SLygYh8Zzav+Km5PttseLHB/JtlrhcRuU9ESsRohLLvYM1No9FoNLEZTEdzALhRKfWVGL1Xl4vIfzHqz7+nlLpDRH4B/AKj1slxwCTztT9G9c9+F67y+/1UVlbi8XgG6DI0vZGUlMSoUaOw2+1DPRWNRjMADJpQUEptxSgTjFKqTUTWYNRiPwmj+QcYFR8/xBAKJwFPKCOb7nMRyRSREeY4CVNZWUlaWhrjxo0jqh67ZhBQStHQ0EBlZSXjx48f6uloNAPOhtot3P3WrznANwknXR98BLApH46QG0fIY77cfGWrZo2tNeZ4EwLJHOLL3qE5tUmA15Nq2afgEBacfNsOjRWLnRKSKkav1n2ALzDqvodv9NUYZYDBEBjRzTEqzXVdhIKIXAZcBjBmTM9S9B6PRwuEnYSIkJOTQ11dXz1xNJpdj7LaZm585VTKkzo4tektjul097p/EAsenNwyOod2i5Aa6lotot0ifGtr4vzaku2aj1fglXQbz2bYcbUrMpq/265x+mLQhYKIpLKt9WFr9M1aKaVEpF91NpRSDwEPAcyePTvmsVog7Dz0Z63ZHfluSyu3vXQO5ekdAJQddC1MPqvnjjYn2JPBkYLV6kD5O6h7di4/3fenXDL9ki67/vPbf3L/N/eT/X9rcFgdCc8lpEK8WfYm9319H9Ud1Rwy8mCu/vlnZJ0+OG7XQY0+EhE7hkB4Win1srm6RkRGmNtHYPSMBaOjU3QXqFEMcJennUFDQwMzZ85k5syZFBYWUlRUFFn2+Xz9Hm/t2rXMnTsXp9PJ3XffPQgz1mg00Swtb+R3z1zH6vRKziSHotQiynxNkDOh5ytjFCRnG8JBhPKWcgDGZ/Q0p+a4cgBo9DQmPJcvq7/k7MVn86slvyLLmcUjRz/Cn/f4JXS6cU6cODAX3I1B0xTEeIR8BFijlPpz1KbXMTo83WH+fS1q/U9EZBGGg7mlv/6E4UBOTg7ffPMNALfeeiupqan87Gc/2+7xsrOzue+++3j11VcHZoIajSYu762p4a8v3kfVqKXMCTr45blvcO3HP6espSyh48P7FWf0bGyYk2QIhQZ3A4UphT22d6eqvYpL37mUvOQ8/njgHzm++HgsYqHtvfcASJoypY8Rto/B1BTmY3TYOlxEvjFfP8AQBkeJyAbgSHMZjPZ/ZRgdlx4GrhrEue1U3nvvPfbZZx+mT5/OxRdfjNdrNAgbN24cN998M9OnT2e//fajpKSnrTE/P585c+bo6B6NZpB5aXkltz3zAu0jXyVXCfec/AI2ZxrFGcVUtFQQDAX7HKO0pRSbxcbotJ4tw8OaQoOnIaH5bGzZSFAFueOgO/jhhB9iEeN27Vm3DkR2PU1BKbWE+PU1joixvwKuHsg5/O6N1Xy3JXYUwPYybWQ6t/xwz4T393g8LFiwgPfee4/JkydzwQUX8MADD3DdddcBkJGRwcqVK3niiSe47rrrWLx4h1vgajTfe4558lpqG3KwdcxP+BhfezVTx/+TMovw1CF/ISt7AgATMifgC/moaq9iTHrP4JZoypvLGZc+Dpul5601WlNIhHqPUcYnz5XXZb133XrsY0ZjSUlJaJz+sksXxNsVCAaDjB8/nsmTJwNw4YUXcv/990eEwjnnnBP5e/311w/VNDWa3YZWj58q31KS00dyxJhTe99ZKRwhN6n+Bura7uUtp3D3tMuYUnxkZJewf6CspaxPoVDWUsbU7Kkxt/VXU6h3G0Ih15XbZb133TqSJg+O6Qh2c6HQnyf6oSI6ekdH8mg0O85XG5uwWDuxU8Yfg3/p+X/l74T2WuiohfY6CLh5N9nF9QV5XFp4EMfMuabL7sWZhn+gtLmUQ0cfGve83qCXyvZKflD8g5jbXTYXybbkxDUFd71xjD05si7U2Ylv40bSTzghoTG2h91aKAwHrFYrFRUVlJSUMHHiRJ588kkOOeSQyPbnnnuOX/ziFzz33HPMnTt3CGeq0ewefFG2GSWKVoLUVX9NvuomFOwuSMkzoodS8iAlj8+avyKtZR1XH/W3HuOlO9LJc+X16WyuaKkgpEIxncxhclw5/dIUemgJJSWgFM4pkxMaY3vQQmGQSUpK4rHHHuOMM84gEAgwZ84crrjiisj2pqYm9t57b5xOJ88++2yP46urq5k9ezatra1YLBbuvfdevvvuO9LT03fmZWg0uwyV5UvBNLeXnHo/+UXz+jxm7b/PZXLOHlgt1pjbizOKI+Gm8Qhv71UoJOXQ6E4sJLXB3dBDKHjWrQMGL/IItFAYVG699dbI+6+/jt1T/qabbuLOO++MO0ZhYSGVlZUDPTWNZrfEHwwhjd9GhMKG5g3M60MohFSIDU0bOGXiKXH3Kc4s5vXS11FKxTXzlraUYhEL4zLGxR0nx5XDxtaNfV4HGJrChMwJXdZ5163HkpyMfdSohMbYHnTpbI1Gs9uweksrI6mILK9vWt/nMZvbNuMOuOM6iMF4+u/wd1DTWRN3n7LmMopSi3BanXH3yU7K7pdPIZaT2Tl5MmIZvFu3FgpDSEVFBbm5uX3vqNFoEmJZRSOFti0AZDozKWnuu87QukbDJDM5O76dPmwS6s2vUNZSxoSMCXG3hzo6GNnppNnbTCAU6HVO3qCXVl9rF6GglMKzfj3OQTQdgRYKGo1mN+LbsmrSbcaT+D75+1DaXNpn0tnaxrVYxcrEzNjJYN6yskgEUllzbKEQCAWoaK1gfGbsasG+TZsoP/U0Zv/+dRSKJk9Tr3MKaxPRQiFQU0OopWVQncyghYJGo9lNUErRvulrOi2GzX9WwaxImGhvrG9az7j0cTHNPu5vv6XsB8eTvGYz6Y70uJpCZVslgVAgpqbgXrGCirPPwbdxI46GNqDvXIVYOQreneBkBi0UNBrNbsLGhk7GetbSatrb98nfB4ANTRt6PW5d0zqmZMe+0bpXrwbAt3EjEzInxBUKpS2lQM/Io7b3P2DjBRdiSU4m49RTEZ8fu1/16VcIC4VwwhuAZ53hH3FO1pqCRqPR9MmXFY3MsJTS4kwhxZ7CpKxJCNKrUGjxtlDdUR1XKPhKDSEQqK2lOKM4rvkoVnXUpmefpfInP8E5cSLjFj2La8YMAFI9/dAUkrpqCvaRI7GmpfV67I6ihcIAM9ClswEOPfRQpkyZEhmntra2xz4LFy5ERHj33Xcj61599VVEhBdffHG7r0ej2VVYvrGJfaxltKfmku5Ix2VzMTptNBua4wuFcHTSlKzYQsFbZmgAgdpaxmeMp8nbFNMfUNpcSkFyAamOVFQoRO09f6b6d7eRevDBjH3icWy5uVgzMgBIdfdd/yi8Pdu1rUubd/26QXcyg85TGHAGunR2mKeffprZs2f3us/06dNZtGgRRx5p1G159tlnmWE+nWg0uzvflW9iHFtpTRpPmsMoDTEpa1KvmsLaxrUA8TWFkrBQqGFC5qGAEWU0K2lWl/3KWsoipqPaO++i8fHHyTz7LAp/8xvEZtxmrZmGUMj22RMyH2U5s7BbjOrIIZ8Pb1k5qUf0qCU64GhNYSewI6Wz+8NBBx3E0qVL8fv9tLe3U1JSwsyZMyPbly9fziGHHMKsWbM45phj2LrVaFfx8MMPM2fOHGbMmMFpp51GZ2cnAAsWLODaa69l3rx5FBcXa41DM2xp7PCR1rgKgDabkz0qBd+mTUzMnMimtk14g96Yx61rXEd2UnaPfACAYFsbAbPVrN80H4GhFUQTUiHKW8ojEUqt/32H1MMPp/CWWyICAcBqViEoDKb22Win3l3fxZ/gKy2FYHDQncywu2sKb/0CqlcO7JiF0+G4O/rez2SgSmdfdNFFWK1WTjvtNH7zm9/EzKoUEY488kjefvttWlpaOPHEEykvN2ydfr+fa665htdee428vDyee+45fv3rX/Poo49y6qmncumllwLwm9/8hkceeYRrrjGKgm3dupUlS5awdu1aTjzxRE4//fT+fFoazU5h+cYmZohh728VxQ+fLqWu4u9MuuoIQipEWXMZe+Ts0eO49U3r4yat+UqNm781K4tATS1jUwpx2Vw9yl1Ud1TjDrgpzihGBYMEampx/vDEHv+jYfNRXiCZsr58Cp76LiWzw+Utdob5SGsKg0ys0tkfffRRZHt06ezPPvss5hhPP/00K1eu5OOPP+bjjz/mySefjHu+s88+m0WLFrFo0aLI2ADr1q1j1apVHHXUUcycOZPf//73kfIZq1at4qCDDmL69Ok8/fTTrDYjLgBOPvlkLBYL06ZNo6YmfjanRjOULNvYyExrGaGsYtoCnbg6AgTq6piUNQkgpl/BH/JT0lwS359gOpmTD9ifQF0dogxHcndNIbrbWqC+AYJB7CN6dlazZGQCkOt39m0+6qzvFo66HnE6cYzpvXT3QLB7awr9eKIfKrqXzg4Gg8yaZdgrTzzxRG677TaKiooASEtL40c/+hFLly7lggsuiDnefvvtx8qVK0lOTo4IIjBiuPfcc8+YgmfBggW8+uqrzJgxg4ULF/Lhhx9Gtjmdzi5jaDTDkWUVTVxmK8My6nDa3CuwewIE6xsYkzYGh8VBSVNP02x5Szn+kD9uJrO3tBRxOEjeZ1/a3voPwcZGijOK+bL6yy77hYXEhMwJBNZtAsBWUNBjPEtKMthsZPpsvUYfKaV6lLjwrluHc+LELuaowUJrCoNMdOlsIGbp7PDfuXPnYrVa+eabb/jmm2+47bbbCAQC1Ncb4Wl+v5/Fixez11579XrOO+64gz/+8Y9d1k2ZMoW6urqIUPD7/RGNoK2tjREjRuD3+3n66acH5sI1mp2Exx+kurKCnFADgREzCZk+sUBjIzaLjeLMYtY396yBFC5vEU9T8JWW4hg/Hpv51O+vqWFC5gRqOmto97VH9itvKSfLmUVWUhb+akObto8Y0WM8EcGakUG6R2jyNBFSoZjnbfO34Qv5uuYo7ITyFmF2b01hGLCjpbO9Xi/HHHMMfr+fYDDIkUceGbH/x+O4447rsc7hcPDiiy9y7bXX0tLSQiAQ4LrrrmPPPffk9ttvZ//99ycvL4/999+ftra2Hb9wjWYnsbKqhT2U8dDVlj+FZNOnHGxqQgWDTMyc2OPpHgx/gt1ij1vV1FtWhmv6XtjNp/5AbS3jJxh5COUt5UzPmw4Y5qNwfkKg2gjeiKUpgOFXSHErgipIs7eZ7KTsHvt0z2YO1NcTrK8naZDLW4TRQmEQGYjS2SkpKSxfvrzPcy1YsIAFCxb0WL9w4cLI+5kzZ3bxZ4S58sorufLKK3s9FqC9vb3HPhrNUPNlRSN7W0pRYqUtawyucKBRKESwuZlJWZNYXLaYFm8LGc6MyHHrGtcxMXNiJOwzmpDHg7+ykoyTTsKWnw9AoLaOCfvuBxiCYHredJRSlDaXcsy4YwDwV9cgSUlYMzNjztWakUFSp/F/1OBuiCkUutc98q43M5l3kqagzUcajWaXZnlFE3OdFUj+NFqVf5tQAAINDUzKNJzN0RVTlVK9lrfwlZcbHc4mTsCWkwMiBGpqGJU2CrvFHnEuN3gaaPW1RsJV/dVbsRcUxO25YE1Px9Hpjxwbi+6aws4qbxFGC4UhRJfO1mh2jFBIsayikWmUQdE+tPpaSfZuC4gINjREIpCinc317noaPY19Rh45iosRux1rbg6BulpsFhtj08dGyl1077YWqK7BFsOfEMaamYG1zQ3Ez2ruLhS869Zhy8vDlt1TqxgMtFDQaDS7LCV17WR4q0gJtkLRLEMoRFWTCTQ0UpBcQJo9rUtY6rom08kcT1MoKwWLBce4cQDY8/LxmyHZxRnFEU0hHHkUTlzzV1dHfBCxsGRkIG3bzEexqHfXY7fYSXcYyW6enVTeIjLHnXYmjUajGWCWVTQxU8y8gZH70uZr62I+CjbUIyJMzJrYpdxFpLFOVrxw1DIco0djcTgAsOXnE6g1spuLM4upbK/EG/RS1lJGij2FguQCI3GttjYSrRQLa0YGqr0Dp7L2aj7KdeUiIqhAAN+GkkHvoRCNFgoajWaXZVlFI/s5K1C2JMjfg1ZvayT6CAxNAWBS5iQ2NG+I5Nqsa1rHiJQRXRzP0XhLS3BM2NYbwVZQQMAsRFmcUUxIhahoqaCs2ah5JCIE6uuNxLXC3oRCJgBFZPaqKYRNR76KCpTfv1PKW4TR0UcajWbY88WmMpKkp039i/JGrnRWILl7g9VOm6+NFJ/h5LXm5RJoNG68k7Im8fz656nprKEwpZB1jevi+hNUIIBv4ybSDjs8ss6Wn0ewsZGQzxfxH5S3lFPWUsbckXMBCFRXG/v2KhQMITQymNGrpjAydSSwc8tbhNFCYRCoqanh+uuv5/PPPycrKwuHw8HNN9/MKaecMqDnWbt2LRdddBFfffUVf/jDHwakGqtGM9z48at3sLTlaTpKbyDky++yzUqQ8cklUHQRAK2+VrIDDiwpFmy5eQTrjRtvuNVmSXMJmc5MKlorOHLskTHP59u0Gfx+HBO2NcwJh6UG6+oYVzgOi1j4tu5b6tx12yKPthpCoXdNIVwUL4XVvWgKe+ftDRjlLbDZcI6P3eZzMNBCYYBRSnHyySdz4YUX8swzzwCwceNGXn/99QE/V3Z2Nvfddx+vvvrqgI+t0QwH/rzkZZa2GFn21x+XzZ5ZXcvHp7esx/aWB4r2BaDN18ZYvw1LWgq2nBwCjab5KFwDqWkDWc4sQioUP5PZ7KHgjDIfhZ3H/tpakouKGJU6inc3Gb1LJmQa+wVqEhEK4aJ4Lho9G3tsD4QCNHmaukQeOYuLEdO3sTPQPoUB5v3338fhcHTJWh47dmyk6mgwGOSmm25izpw57L333jz44IMAfPjhhxx66KGcfvrpTJ06lXPPPbfPWkP5+fnMmTMHu71n8o1Gs6vz3w3f8OiGP2ILGdVCR+WGOHxqQZfXbLtZsXTkNqGQ5rdiSU3BlpNN0CwRk+HMIN+VT0lzSZ+RR16zh4JjfE9NIVCzza9Q3VEdeQ+GpiBJSVgyYvspYJtQyPY7afQ09vgfb/I0oVCRjms7s7xFmN1aU7hz6Z2RJhoDxdTsqfx8v5/H3b569Wr23XffuNsfeeQRMjIy+PLLL/F6vcyfP5+jjz4aMLKeV69ezciRI5k/fz6ffPIJBx544IDOX6PZFahorOVnH12HiJN/HfMgC949NbYNvmo5JGVAttnLwAxJtaakYs02NAWlFCISabiTak+NdGWLhbesFNuIEVhTUyLrbFGlLgDGZ47nw8oPcVgcFKUaBSv9NdXYCwvjJq4BEYGR6bXiD/lp9bV2cXZH5ygEW1oIbN2608pbROa4U8/2PeTqq69mxowZzJkzB4B33nmHJ554gpkzZ7L//vvT0NDAhg1GqNx+++3HqFGjsFgszJw5k4qKiiGcuUYzNHT6vZz92lUELS3cMucuZhVNItmWHDtap+orGLkPWIxbWZuvjWQPWFJTseXmoDwelFkgb1LWJEqbS/mu4TsmZ03GIrFvf77SMpzFxV3WWTMzwW4nUGvkKkzIMExG4zLGYbVYAQhsre7VyQzbGu2kewzB0V3QRYRCci6+TUbFVUe3uQw2g6YpiMijwAlArVJqL3PdrcClQJ2526+UUv82t/0S+DEQBK5VSr29o3Po7Yl+sNhzzz156aWXIsv3338/9fX1kVaaSin+9re/ccwxx3Q57sMPP+xSptpqtRIIBHbOpDWaYcQ5L/6KDss6Tim6kdP3PACWPUaOz0PDykWw7NWuO9ethfnXRRZbfa04vSEsaWlYs40qo4GGBhwpKUzMnIgv5OPbum85Y/IZMc+tQiG85eVknn5al/Uigj0vD39UWGr0XzCqqKbsv3+v1yZWK5b0dJLdhtmowd3QZYxoTSFQZ5S3sOXl9xxoEBlMTWEhcGyM9X9RSs00X2GBMA04G9jTPOYfImIdxLkNGocffjgej4cHHnggsi7c3hLgmGOO4YEHHsDvN+qfrF+/no6Ojp0+T41mOHLz2w9R5nuHqa4TuL14PDx4MCy+jhwsNDqSIGdC19e0k2HmjwDjgavV14rDEzB8CrnbhAJsczYrVFx/QmDrVlRnJ87iCT22GbkK2xLYHBZHZJxI4lph/GzmMNb0dJLi1D8KC4WcpBwC9ca5bHk7txTOoGkKSqmPRGRcgrufBCxSSnmBchEpAfYDYrciG8aICK+++irXX389d911F3l5eaSkpEQqoV5yySVUVFSw7777opQiLy+vz+ih3/72t8yePZsTTzyxy/rq6mpmz55Na2srFouFe++9l++++450U0XVaHYlXlj1Cf/e+g/yQhN5NlAOT5wImWPgjMfJrvmATW2b4KSn4h7vDrgJhALY3WL6FIy8hqApFIozirGIhZAKxc9kLjPKVzgnxhAK+fmRiqUp9hSe/+HzEX/CtsS1+HWPwlgzMpB2I8Ouu0ms3l1Pmj2NJFsSbaaTfGfVPAozFI7mn4jIBcAy4EalVBNQBHwetU+luW6XZMSIESxatCjmNovFwh//+MceTXAOPfRQDj300Mjy3//+98j72267LeZYhYWFkZaaGs2uzqPLXsJKiFerlmCzOuCIW+CAq8CeRE7rCr6ujV1+Pkybrw0JKayeAJa0NKO6KduympNsSYxJG8PG1o29lLcwI48mxBYKHUuWRJbDoahgaBhAYppCRga0d2ARS0yhEG6uE6yvx5qZuVPDUWHnO5ofACYAM4GtwD39HUBELhORZSKyrK6uru8DNBrNsKe2zUNm6zKygwEy9j4LrvkKDroB7EkA5LhyaPY2EwjF97O1+lpxmcXwLKkpkSfsYOO2G+/03OlMzppMsj055hi+0jKsWVnYsrJ6bLMX5BPq6CDY3tPc21vHte5YMzMItbSQ5cyi0dPYZVt0iYtAXf1ONx3BTtYUlFKRzu8i8jCw2FysAqLjw0aZ62KN8RDwEMDs2bN102CNZjfg9a82k2Wtpd2WCif9vcf2nKQcFIpmb3OX3sXRRBfDs6amIg4HlvR0AvXbhMKv9v8VvpAv5vFgaArRmczRbGu2U4s1tWuGsd/suNZbhdQwlowMgi0t5LhG9NAUGjwN7JG9h3Ge+nqsQ1BaP2FNQURii9Z+ICLRYvQUYJX5/nXgbBFxish4YBKwdEfPp9Fohj9KKdYtfQefNUhmSuxIm3CHsnhF5MDMUTCFgiU1DcDMat52TKojNWa3s/A8fKWlMZ3MALb8rrkK0QSqaxCXq9fEtTDWsFBwZMd0NEe34bTl5vU53kDTp1AQkXki8h2w1lyeISL/SOC4ZzEcxVNEpFJEfgzcJSIrRWQFcBhwPYBSajXwPPAd8B/gaqVUcHsvSqPR7DqsqmplZst7tFitZGSMiblP2M7em1Bo87VFeilYUlMBsOZkR+of9UWwsZFgS0tMJzNEawo1PbaF+yj0lrgWxpqRCaEQhZLR5Xo6/Z10+DvIdeWilDKFwvA0H/0FOAbjaR6l1LcicnBfBymlzomx+pFe9v8D8IcE5qPRaHYjXlpWzk+tS3nCWcCkOE/xOUmmUIhTWRRMn4LZdS2cjWzLzok4j/siUt4irqawzXzUnUB1da99FKIJJ7AVBFJo8DREMq7D15bryiXU0Ylyu4dEKCRkPlJKbe62Sj/FazSaHcYbCFLzzTtkSRutFiHdGTucOqwpdHfMRtPFfJRmmo9ycyIhqX2xrRBebJ+CNTUFS0pKJIEtGkNTSFAoZBomptxgEt6gl86AkccU1hpyXbkEhyhHARITCptFZB6gRMQuIj8D1gzyvHZpampq+NGPfkRxcTGzZs1i7ty5vPLKKwN+noULFyIivPvuu5F1r776KiLCiy++OODn02gGmvfW1HJ44GO89jTag95IC8rupNpTsVvsvfsUvK1kBIzwzYj5KDuHYHMzykwW7Q1vaRmW5OReS1XY8vMjRfHCqECAQF1d4ppCuCiez5hr+Jrq3IYgyHXlGnkPMGw1hSuAqzHyBqowwkmvHsQ57dKES2cffPDBlJWVsXz5chYtWjRo+QTTp0/vkhPx7LPPMmPGjEE5l0Yz0Ly6rIzjbMvo2OM4gLid0ESEHFdOr+ajNl8bWUGjVIwlxRAKkazmpqY+5+IrK8UxYUKvfgGjLWdXoRBJXEtUU4gqigfbTGKRbGZXTkQoDLvoI7PUxF+VUucqpQqUUvlKqfOUUonpY99DdmbpbICDDjqIpUuX4vf7aW9vp6SkhJkzZ0a2L1++nEMOOYRZs2ZxzDHHsNVMsnn44YeZM2cOM2bM4LTTTouU4liwYAHXXnst8+bNo7i4WGscmkGjttWDlLxHKp20TzK6nMXTFMDwK/TlU8jw20AES4oRLNk9q7k3vCWlPQrhdcdWkE+gpqujOdxxzZ6gphCOUEr1GMthTaHeXY9FLGQ5swjUmZpC3s6PPurV0ayUCorIWBFxKKXiB/cOU6r/+Ee8awa2dLZzj6kU/upXcbfv7NLZIsKRRx7J22+/TUtLCyeeeCLl5UaNeb/fzzXXXMNrr71GXl4ezz33HL/+9a959NFHOfXUU7n00ksB+M1vfsMjjzwSEVxbt25lyZIlrF27lhNPPJHTTz+9X5+RRpMIr3xdxfGWTwkmZdOSPxXoQyi4cqjrjJ+wavRSsGFJTY087XfPao5HsK2NQG1tzEzmaOz5+bTW1UWcw2D4E6D3NpzRhDWF5M4QpG8TCg3uBrKTsrFarIamYLNF9t2ZJBJ9VAZ8IiKvA5FUPqXUnwdtVrsRV199NUuWLMHhcPDll1/yzjvvsGLFisgTeEtLCxs2bMDhcERKZwOR0tmJ9FM4++yzue+++2hpaeGee+6JlNBYt24dq1at4qijjgIMLWWEmXG5atUqfvOb39Dc3Ex7e3uXqq0nn3wyFouFadOmUVPTM/xOo9lRlFIsXlbCC9avsO51Lq2mszWeoxmMXIW1DfEf8oz+zJaIPwG2CYXorOZY+MI1j+I4mSPj5eeD30+wuTmS9RwWCr11XIvG4nQiSUk4OnxIunQxH23LUajDlpODWHZ+d4NEhEKp+bIAaYM7nYGltyf6wWIoSmfvt99+rFy5kuTkZCZP3lbTRSnFnnvuyWef9awruGDBAl599VVmzJjBwoUL+fDDDyPboueRiAlLo+kvKypbGNvwMUkOL+x1Gq2+VgAyHPGfjHOScmj0NBJSoZi9EFp9raR4pUtzHGtYU+gjV8FbGhYKvWsKkQS2mpqIUAhsrTYS1/pRiNKakYFqbSNzXGYX81E4ymqochQgAUezUup3SqnfYdQpuidqWRODoSqdfccdd/QosjdlyhTq6uoiQsHv97N69WoA2traGDFiBH6/n6effnqHz6/R9IcXl1dyku1zQqmFMGYuLd4WoHdNIceVQ0AFaPW2xtxulLlQXTQFi1nuoi9NwVtagtjt2E1NPR6xchX8NTV9dlzrjjUjg2BrC9lJ2V00hTyX4UMI1g1joSAie4nI18BqYLWILBeRPQd/arsm4dLZ//vf/xg/fjz77bcfF154YZfS2dOmTWPfffdlr7324vLLL+9TI/jtb3/L66+/3us+xx13HIcddliXdQ6HgxdffJGf//znzJgxg5kzZ/Lpp58CcPvtt7P//vszf/58pk6dugNXrNH0D48/yHvfbOAw6zdY9joVLNaIptCXoxli5yoEQ0Ha/e1Gg50ooSAiWHNy+tQUfKVlOMaPR2y9G0/sBT2FQmDr1oSqo0Zjzcgg1NxiRFS5GwipEA2ehi4lLqxDkKMAiZmPHgJuUEp9ACAihwIPA/MGb1q7NjurdPaCBQtYsGBBj/ULFy6MvJ85cyYfffRRj32uvPJKrrzyyl6PBWhvb495bs3uS2OHj9Agmg3fX1vLXN/n2Bx+2MvocNbqbSXJmoTDGr9MdKTUhaeBYrra/tv9xu/U4fZjTUvtss2Wnd2l/lEsvKWlJO3V97Ou1YwG8kf52vw1NaTMndvnsV3GyczAV7GRnKSprGpYRau3lUAoYJS4CIUINDQMmaaQiFBICQsEAKXUhyKS0tsBGo1m1+TCl+7iy/p36Cy/flDP86zrC1TGGKRoFmD4A3ozHUHvRfHCJiWb2x/JUQhjzc3ptf5RyOPBX1lJxkkn9Tlvi8OBNSsr0oFNBQIJd1zrMk6kUqqhKYQT13JcRrIdwSC2nOErFMpE5P+AJ83l8zAikjQazW5ERX0HS7eswJZew60nTsEqg1NZ3+lr4oAPVyB7XgOmHb7F29Kr6Qi6agrdafUbQsHS6e1iPgKz/tG69XHH9ZWXg1J9Rh5FxisoiOQqBOrrIRRKqONaNNYoodAZ6KSyzUhuzU3KjcpRGL5C4WLgd8DLgAI+NtdpNJrdiLvfWYfNbjh8jxuxmYI4xel2mPX/BRWMmI7A1BT6EAqZzkysYo2rKViDCovXh6W7+cisfxSdWxBNOPKorxyFyHj5eRGfgt9MBrX316eQnoHyeskV45rXNa0DzLpHZVvMeQ9ToWC2y7x2J8xlwIj35WsGHh2yunuwsrKFxSu2MmnKFqqBhmfPoMDXd72g7SZ3MhROjyy2+loZmTqy10MsYiErqWe3MjAij5LM9FprN03Bmp2D8vsJtbVFKpRG4y0tAYsFx7hxCU3dlp+PZ41R/i2sMdi2Q1MAyA0Y4d/rmwxNxqh7tMIYc7gKBRH5L3CGUqrZXM4CFimljun1wCEiKSmJhoYGcnJytGAYZJRSNDQ0kJSUNNRT0ewgd729lr1cjdTiASw0HP4ryJwyeCcs2CtiOgJDKOzh2KPPw7KTsmNrCtEVUrv5FGw5hsYTaGiIKRR8JaU4xozBkmAvZHt+AcH6BpTfH5W41k9NwayUmmUWxVvftJ4kaxIp9hQa68J1j3Z+iQtIzHyUGxYIYGgOIhK7PdIwYNSoUVRWVqL7N+8ckpKSIlnYml2TJRvq+XhDPS8Xv8uFZgZtQ24xTDy+z2OVUvy7/N8cNvqwuH2PE6HF29Knoxni1z9q87VFlc3upilEspobYfz47ofiLSvDEaexTixs+fmgFIGGBiNxLTm5X4lrsE1TSPcan/em1k0UpRYhIgTq641kuJQdbna5XSQiFEIiMkYptQlARMZi+BaGJXa7nfExvniNRtOTUEhx53/Wsk9GB5n1/4Ei43mvtxLV0ZS3lvOLj3/Bj/f6MdfNum675uAP+XEH3H36FMBwNm9q29RjfauvlRSfBQj2MB/ZeslqVn4/vo0bSTviiITnawvnKtTUGIlrCXZciyYsFFLdxq1Uobq14cwdMktHIoU1fg0sEZEnReQp4CPgl4M7LY1GszP496qtrKxq4U8jPqDBuu120Fs10mhqOgyb+gvrX6DT39nH3rEJh5MmJBSSjBDO7r6sNl8bOSEXQM/oo17qH/k2bYJAIG4LzliEs5r9tbUEtm5NuDpqNGGhIG0dpDmM6kFd6h4NkT8BEitz8R9gX+A54FlgllLq7cGemEajGVz8wRB3v72OuXl+Jmx+idpio/iiRSwJawrh+PpWXyuLyxZv1zwidY/i9FKIJtuVjSfoiXQri4zhbSUraPi2LKldS7RZwzWKYmgKfbXgjIU9qtSFv6YGW4J9FKIJl88ONrdEMrXDIbfBIax7BL0IBbNkdgaAUqoeo0Lq0cAFIpKYR0aj0QxbnvtyMxUNndwx8n9IyE/92AMAKM4oTlhTCJeynpg5kSe/e5KQCvV7HpG6RwlqCtDTvNXqbyXLH+661jW3Vmw2rJmZMbOavaUlADiLEzc5W3NywGolsHUrgdra7dIULCkpYLVGchUgSlOoqx+yHAXoXVN4HkgBEJGZwAvAJmAG8I9Bn5lGoxk0On0B/vreBo4YI4wpWwTTz6BWQrhsLsakjUlYU6h315NsS+aS6ZdQ0VrBkqol/Z5LpO5RIo7mOAlsbb420gOGi7S7TwHMrOYYPRV8pWXYR47Ekpy4U1csFmx5ebhXr4ZQaLs0BRGJFMULC7pcVy7K5yPY3DwkHdfC9OZodimltpjvzwMeVUrdIyIW4JtBn5lGo+lCaUMt937yBuOT+u6x0Rfratqoa/Py+z2WILVuOOhG6tY8Qq4rlxxXDt/UfZPQOHXuOvKS8zh67NH8edmfeeq7pzh41MH9mksixfDCRIriubve4Fu9raT5bWC1Ii5Xj+Ns2TkEYnRf85aW9ivyKDJefj6elauAxDuudSec1ZydZAiAXFcugUbjuobSfNSbUIh2fR+O6VxWSoV0/L9Gs/P51TsL+c73JG+W2lD+Hb9pXLRvJiPWPgnTToK8KdQtryPPlUeOK4cmTxOBUACbpfcAxbrOOnJduditds7Z4xz++tVf2dC0gUlZkxKeR9jRnJBPIVz/KIamkOJL79J1LRpbbg6e79Z0WaeCQXzl5aQccEDCcw1jL8jHs8JMMtsOTQHMSqktLeS4jHyQXFcuga1miYshylGA3oXC+yLyPLAVyALeBxCREcAu15pTo9mVqW3zsKK6Els2/OuS0Rw+5vAdH/TDO+C7Njj4JsB46p+aPZWcpBwUimZvc8TOHY96dz3TcqYBcMbkM3jw2wd5as1T/G5e4i1XWnyGTyEchdMb2a6eRfGUUkbymi8da0rsWp3W7JzIU3gYf1UVyuvtV+RRGFvetlStHdEUAnV1jE0fi81iY0TKiG3ZzMPUp3AdRr2jCuBApVQ4570QI0xVo9HsJB7/tAJlaQOgtLl0xwf0tMLn/4Apx0PhXoDx1B/WFCCxXIU6d11EcGQ4MzhxwoksLl2csE8CDE0h2ZaM3WLvc1+7xU6GM6OLpuANevGH/CR5FJa02ILFlpNNqLWVkG/b86y31Iw8SrDmUZfxzAgkSU6Oe86+sGSkE2xp4eixR/PGyW8YTYTqw5rCMDQfKSMQuEdTAKXU14M6I41G04V2b4AnP9vIiPFu6oHSkregYweV9S1fg6cFDv4ZAB3+DjoDneQl58WN8OlOh78Dd8BNXvI2U8e5087l+fXP88L6F7hixhUJTSWRstnRhNtyRh8P4PQEsaTGNkFFZzVbzF7KPlMo9NWCMxa2AqOsRX87rnWZU0YmwZYWrBYro9KMqgBBUyiE5zsUDE5tXI1GM2A89+VmWj0BJssm6hWU1q+GFe/t+MBTT4CifYFtoaVdNIU+wlKjjwlTnFHMgUUHsmjtIi7e6+Jem+aEafW19tqbuTvhHgRh2nyGBmX3+LFkxzYfRWc1202h4C0pxZaXF7MeUl/Y8o1r7m/No2isGRmE2ttRgUCk41ugrh5rRkbCdZgGAy0UNJphjD8Y4tEl5Rw21k6VvxlsVsqTUwneXI7VYt2xwaNs+OEktP5oCuFjuvsdzp92Ppf/93LeKn+Lkyb23bim1ds/TSE7KZu1jWu3HW9qCrZOP9bUeOajnlnN3rKy7TIdAdhNTaG/1VGjCWc1B9vasEUS7IauDWeYRMpcRBCRLBHZe7Amo9FouvLmiq1UNbv5Vd4nNFqETHsa3qCPLYEOSMrYsZdl279/+Kk/35VPij0Fp9XZp6ZQ7zZMHdGaAsDcEXMjyWyJlFZPpJdCNOFSF2HCmoKl09OjxEWYsDkmYOYqKKXwlZZul+kItvkUwlrH9hCulBpsbo6sM+oeDV3kESQgFETkQxFJF5Fs4CvgYRH58+BPTaP5fqOU4sGPytgjz0nhxmfxWizMGWmET5Y0lwzouaI1BRHpceONeUzntmOiERHO2+M81jWtY1nNsj7P3ertp1Bw5dDub8cbNMqihjOipdPdI5s5jC3biFoKNhiCLFBTQ6ijA0eC3da6Y01PZ+Q9d5N51lnbdTxs0xRCLS2RdYEhLnEBiWkKGUqpVuBU4Aml1P7AkYM7LY1Gs6SknjVbW/nd+O9oNG/QswtmA1DaMgARSFHUddaRZE0i1W48aee4ciKaQNxj3HU4LI6YN/Tji48nxZ7C2xV9l0lr9bVGchRUqO8yGd0T2Np8bViDCrw+rHEigSwpKYjLFdEUwjWPnBMm9nm+eGQcfzz2gu3vIhAxH3UXCkPoZIbEhILNzE04E9i+ilcajabfPPi/MvJTHcze8hSNecbNa1z6OPKT8wcmLDWKWndtREuA+H0LoglnM8eKvkmyJTEqdRTVHdW9juENevEEPaQ70ml6/nlKjjiSUEdHr8d0T2DrrcFONLbsbAKmpuArCwuF7dMUBoJwD4awUAh1dKA6O4c0RwESEwq3AW8DJUqpL0WkGNjQ10Ei8qiI1IrIqqh12SLyXxHZYP7NMteLiNwnIiUiskJE9t3eC9JodgdWVbWwpKSeW/bYiqV+HQ1TjwOM5K2JmRMHXCjUu+u7+Aa6R/jEPKazvtfktsKUQmo6a3odI7pstmfVagJbt9L8yqu9HtM9j6LN1xZVITW+UIiuf+QtKcWakTGkoZ/WzEzAqJQKRHIUhrLuESRWOvsFpdTeSqmrzOUypdRpfR0HLASO7bbuF8B7SqlJwHvmMsBxwCTzdRnwQGLT12h2Tx7+uIwUh5VjWl+AtBE05k0GjCf44oxiylvKt6siaTzqOuu6+Aayk7Jp8jYRDAXjH+Ou6+FkjqYguaBPTSG6GF6gthaAxieeQAXjnzcsFMK5Cq2+VnJDRkG7eD4FMOsfmVnNRs2jiUPasjds6gprCtsS14a/o/ku09FsF5H3RKRORM7r6zil1EdA97KEJwGPm+8fB06OWv+EMvgcyDRNVhrN947Kpk4Wr9jKdXu5sW38CPa/ggazFERmUiYTMyfiCXqoaq8asHPWdtb20BRCKkSztznuMdHZzLEoTCmk2duMJ+CJu0+kl4Ijg0BtLZKcjH/TJto//DDuMd3NR22+NnKCRhG8eD4FAGtONsH6eiPyqKQEZ/HQmY7AKOltSUsj2Gp8BgGzN/NQm48SyVM4Wil1s4icglHy4lSM7mtPbcf5CpRSW8331UA486MI2By1X6W5bivdEJHLMLQJxowZsx1T0Gh2HqtrNnPe4kvw1h8JHdMTOiYQVAhwbugNcKTCrAU0fPt3Mp2Z2C12JmQaYZRlzWWMThu9w3OMzmYOE53AFn4fjSfgoc3X1iPyKJqCFOPfu7azljHpsf9XI70UnOn462pJP/poOpZ+QePCx+O2yHTZXCTbkiPmo1ZfK+MDTqAPn0JOLoGmJoINDQRbWrar5tFAY1RKbQYYFiUuIDGhEN7neOAFpVTLQKhcSiklIv3u9ayUegh4CGD27NnDtle0RuPx+7jozZ8SsG1hzJi1HJR+fMLHzs7qJPnd12C/y8CVSaOnMfKEXJxpPOGWNJdwyOhDdniesTKTuySwZfU8Jl6OQjQFyYZQqO6ojisUwppCmiWFzoZG7CNHkH3e+dTedRfu1atx7blnzOOifR5tvjYyArEb7ERjy8mGQIDOr74C+tdtbbAIl88Gow0nVmvE1zBUJCIUFovIWsANXCkieUB8fbB3akRkhFJqq2keqjXXVwHRjzyjzHUazS7Lgldvx23dQI6jiFa1hp8fN7nPUtQR3vkNKAX7G/WDGtzbntjTHenku/IpaykbkHlG5yiE6avURVgo9GU+Anp1NocdzantATpDIWz5+aQffzz1f/87jY8/TtFdd8U8Ljo6qs3XRro/E+jLfGRcU+fSLwGGiaaQTsh0NAcbGrBlZyPWHcxU30EScTT/ApgHzDYrpXZi+AC2h9eBC833FwKvRa2/wIxCOgBoiTIzaTS7HH/77DVWd77KGPvh/HLu9bT521hVv6rvA8GoYLr8cdjzZMgaC9BFUwBDWxioBLY+NYVYx8QQJN3JTzZi+HtzNoc1haRmo+eyLT8fa1oaGaedRuu/38JfUxvzuBzXtqJ4RoMd40baW/SRLSIUlmJJTsa2A9nIA4UlWlOoG/oSF5CApiAiycBVwBgMW/5IYAp95CyIyLPAoUCuiFQCtwB3AM+LyI+BjRi5DwD/Bn4AlGAInYu241o0mmHB8qpSHlrzB+xqFM+cfht8+DsswKdv38BMawLJTh214G2FuT+JrGpwN0Ru1GD0RH5pw0uEVAiL9KtaTQ9i3eDTHenYLfa4mkJYkPSmKbhsLjKdmb1qCi3eFtLsaYTqjfOE+xRkX3A+TU89RdMzz5B//XU9jstOyuarmq8IqRDt/naSfQI2G+J0xj2X1cxq9q5fT9L06UMaeRSmq/lo6LOZITHz0WPAcgxtAQyzzgv0IRSUUufE2dTDe2SW6b46gbloNMOaDq+Xy9++HkTx9yP+QsaXD8CXj7DXmPF8Emrkqg5/34MAzLkkUsHUF/TR5m/r4vAtzizGHXCzpX1LpOzy9hLOZk6zbzO9iEivuQr17nqsYu2ivcSir7DUcNnscDhquKaQY/Ro0o48guZFi8i94nIs3Vps5rhyaPY20+xtRqFI9iqscbquhYm+4W5vzaOBxpqRSbC1FaUUgfp6nJMnD/WUEhIKE5RSZ4nIOQBKqU4ZDiJWoxmGnPfK/+G1lnP++P9jnsMHH/0Jpp/J3OIZPLzyYVouWpxQ28lowmaS6BvwxEwjw7mspWzHhYIZWtr937q3rOY6dx05STl9ail9JbCFi+EFqmrBYjGcwSbZF15I23/fpeW118g6++wec1MoNrVuAsDpDfVqOgKzrITFAqHQdtc8GmisGRkQDBJqayPQ0DAsNIVE9E6fiLgABSAiEwDvoM5Ko9kFueuj5ynxvsVE53HcPP8UePUqcGXDcXcyv2g+IRViafXSfo8bvjFHm4+KM4yb2kBkNte56yL2/87ly6l/+GHjfK6cSH2hWMfkJvd9AytILqCmo3fzUbozHX9tLbacnEhfAQDXrFkk7bUXjY8/0aMmUlhrKm8pB8INdnoXCmK1RkxIO1LzaCAJ1z/ybdoMfv8uIxRuAf4DjBaRpzEykW8e1FlpNLsYn2xcw5Olf8IZHMdTp9wOH98DNSvhh/dCcjZ75e5Fqj2VT6o+6ffYYRNOuD8xGK0v81x5A+JsruvcloTW/Pzz1N37V1Qg0Gul1LrOOvJdfftHClIKaPI2xU1gi2gKtbUR01EYESH7wgvxlZfT/tFHXbaFBWRFawUANncAax9CAbZVSx3KmkfRhMtn+0qN73GoE9cgseij/2IkrC0AnsWIQvpwcKel0ew6VLc1cfW714Cy8OAx95LStN40G50BU43cBLvFzn6F+/HZls8S6jEQTdh8FK0pgOFXKGve8bDUaE3Bt7kSgkH81dWRCJ9Y5TTq3fUJaQrhsNTazthRROGy2YHauh5CASD92GOwFRTQ+PjjXdaHTWkVLRUA2Dq9fWoKYGQ1i8OBfdSOmdwGinDXt3DV1l1FUwBIApqAVmCaiBw8eFPSaHYdfIEAZ7x0DQFrHTfO+AOzCsfAq1eaZqOuMfbzi+azpWMLG1s39uscEU2hm1N3YuZESltKd6gGUqe/kw5/R0RT8FdWRv7mJOUQUIFILkEYf8hPo6ex18S1MNEJbN1RSnVxNMcSCmK3k3XuuXR+9jm+iorI+oj5qNUwH/XWYCca18yZpBx44JDnAoSxmOYjr9kveqiL4UFiIal3AmcBq4Hwr09hlLrQaL7XLHj1NprlW47Ov4IFs46ED++E6pVw9jOQ3PUmPnfkXAA+2fIJ4zLGJXyOBk+DUdrBntxlfXGGEYFU3VHNyNSR2zX/cDhqfnI+IY8nEgXkr6wkZ9a2BLbMpMxt8zGFVG/hqGF6S2BzB9z4Q34yJZVgY2Ok73F3Ug8+iLo//xnPmjU4xo0z1tlTcVgcbG4zq+N0urGk9S0U8n/60z732ZlYMzIB8EbMR0NbDA8S0xROBqYopY5XSv3QfJ04yPPSaIY9f/jwGVZ2vMIY+2HcfcyVhjD46K4uZqNoRqeNZnTaaD7b8lm/ztM9cS1MOAJpR/wKYbNOrisX/5YtkfU+U1OAnglsiZS4CNNbAls4cS3bbUQ9xdIUABxjx4II3rJtprJwyGwgFMAqVlRbe0I+heGGNcMwH/k3VyJJSVhS4pfp2FkkIhTKAPtgT0Sj2ZV4Y82XPFt+N67gBJ4/7U9Ygt64ZqNo5o2cx9LqpfiDCeYr0LXERTTRhfG2l/ANPj85H//mbTUp/ZVVcUtdxGvDGQuXzUWGMyOmphCpkNpqGCDscYSCxeXCPnIkvrLyLuvDgjLLkory+3sthjdcsSQlIUlJEAphy+0ZFjwkc0pgn07gGxF50GyEc5+I3DfYE9Nohisb6rfy609vxKJSePKH/yBl8xL4x1xDUzjhLz3MRtHMGzkPd8DNN3XfJHy+Rk9jDyczGBFIua7cHWrNGa0p+Ex/gmPChIhPAXpqCmGTUyLmI4DC5MKYYalhX0VaiyEg42kKAI7iYrzlXYVfWGjlhoyn60TMR8ORcFjqcHAyQ2JC4XXgduBTjMzm5UDf3bg1mt2QDq+Xc1+/mpCljf8389dM+d//wVOngljggtdgjxN6PX6/wv2wiY1Pt3ya8Dkb3A1xM4cnZEzYoVyFenc9TquTdEd6xIThmjEDX1Ul6c50bGLroSnUu+sRJKb2EouClAKqO3uaj1rMHhHJLUa4am9CwVlcjK+8oku+QlhohRvs7IrmI9gWgWTNHdrezGESEQqZSqnHo1/ELKar0ez+nP/Kb3FbN3B1yjyOf/syWPM6HPILuPJTKD60z+NTHansnbd3wkIhpEI0eZvi3oCLM4spbS7td5hrmNrO2kg2s7+qEvuoIhyjRxGsqwevj+yk7JiaQlZSFnZLYlblvjQFZ1MH2GxYs+LfVhzFxSi3m0D1NuES/kyyE2jFOZzZFTWFC2OsWzDA89Bohj1/++w1Nnj/zQ877Vz53VMwYm9DGBz2S7AnJTzOvJHzWNOwJpJ/0BvN3mZCKhRXU5iYOZHOQGefbS/jUe+u75Kj4CgaFYnh91cZfoUemkIfvZm7Ey+BLexTsDW2YcvLQyzxb0fO4vEAeEu3mZAiPoVwg53U+GWzhzOWzLBQGPrII+hFKIjIOSLyBjBeRF6Pen1AzzabGs1uzZraSh5a8/8Y6xN+21wHJ/8TLnwDcif1e6x5I+ehUHy+5fM+9w0/pcfVFMLlLrbTrxBuw6mUwr95M/bRo7EXmUKhspJsV2xNIZHIozDxEthafa0IAvVNccNRwzjM1pm+KL9C2HyUHjA0lt4a7AxndiVN4VPgHmCt+Tf8uhE4ZvCnptEMDwLBIBe/eQM2cXNfbRVJJ/0dZp4D2xkpMi1nGhnOjIRMSPGymcOEw1K3169Q564jLzmPYHMzoY4O7KOKsI8qAraFpfaIPuqjN3N34iWwheseBetq40YehbFmZ2PJyOgSlhppOuQ30q12WZ+CmaswHEpcQC/Ja0qpjRg9D+buvOloNMOPKxf/mXbLGm6pb6J42hkwbXt7TBlYLVYOGHFApORFb2GIEU0hjlDITMokOyl7u4RCOJs5z5UXyWR2jB5tmHKcTiMsdapR/yg8z5AK0eBuiJiclFK0f/AhKfvvFzfGPiwUuoelhuse+WvrSJ6zX69zFRGc48d3CUsNfyapfuPZ1tJL17XhzC6jKYjIEvNvm4i0Rr3aRKQ13nEaze7Eiys/4fOmpzi4U3GqJQOOu3NAxp03ch617to+E89ilc3uTrjcRX+Jbq4TFgr2UaMQEexFRZGwVH/IH7H/N3oaCapgRFPwrt9A5VVXUXXTzT0qmYYpSIkvFLIllVBLS6+RR2EcE7qGpY5IHUGaPY28oBmSuotqCrb8fBDBVjhiqKcC9G4+OhdAKZWmlEqPeqUppdJ30vw0miFjS2sjty/9NZlBK3+sq8Jy8j8hqX+9EOIxb6TRs6ovE1KDpwGb2Eh3xv+XK84wCuP1NwIpug2nb7MpFEx/gn1UEb6qyh4JbJFsZjNxzbt2DQDt779Pw0MPxTxPOIGtu/mozdvGCLfhJE5EKDiLiwnW1RNsNQRUij2FD8/6kHG2AsRux+JwJHjlw4v043/AuGefwV6QQFe+nUBvQuGV8BsReWknzEWjGTaEQiHOf+3nhKyN/K12Mxn7XwXjDxqw8QtTChmXPo5l1b2n/IRLXPTWzGZi5kTa/e29NrOJRURTMM1H1uxsrKaz1jFqlGE+6pbA1r2fs2f9esRuJ/0HP6Dur/fR/vHHMc8VKyy1xddCgdu4kSekKYw3nc1RfgWH1UGoo32XNR0BWBwOXDNnDvU0IvRWEC/a0Dk8io9rNIPIpuY6lm5ez8raEr6q/Yba0Odc1upjZvoEOPz/Bvx8e+TswTe13/S6T4O7oUsfhVgUZ25ruBOO9EmE6HIVLZWbu5STtheNItTaSrYZ7tldU4iYj9atxzFxIiP+8Hu8ZWVU/ewmxr/4Ao7Ro7ucK1YCW6u3lZx2Y759RR9BVFhqWXmXm2iorX2XNR0NR3oTCirOe41mt+HKN/7Ml3Uf4qUGrJ2R9UoJc7zJXNm8FS59oV95CIkyOWsyb5W/FXG4xqLB3RDXyRwmHJZa1lLG/KL5CZ+/zl2Hw+Ig3ZFOXWUVrr32imwLC4iMBm9kHuFjIMp8tG4dKfPmYXG5GPW3+yg/7XQqr7mWcc8+06WvcmFyISvrVkaWw2WzM9uNW0tf0UeROdntXcJSAULt7btsOOpwpDfz0YywYxnYWzuaNbsbL6/+jCWNj6EIMDZpLgdmX8SPJ93OP2b/jWWjzuCxrWuxHfYrI0ltEJicZTRp39C0Ie4+8SqkRpOdlE2GM4Oylv4VxguHoxIK4d+ypaumYIalJtW2YBFLF/NRmiMNp9VJoKmJQF0dzilTACNyqejuP+Fdt46tt9zSxcfRPYGtM9BJUAVJbw0gDkekr0BviM2GY+wYvN0K44Xa27Huoolrw5HeQlKHRxcKjWaQuHvpX0Gl8O8zn6XAUwtrF8Pqv0GVaeefeCTMH7z6+1OyjJvpusZ1zCqY1WO7UooGT+wKqdGISMTZ3B/qOo0ktEB1NQQC2EdvEwoOU0AEq7aSlZIViYKqd9dH/AnedesBcE6eHDku9eCDybv2Gur+eh+uvWeQfd65wLaw1NrOWsakj6HFa9Q9SmnxYsvPT7g6qLN4At4NXYVosL0d+8jt6yeh6UmfTXY0mt2RJ79+nzbLak6z7EvBwqOgfp2xYeQ+hv9g6gmQN2W7E9QSIT85nwxnBuub1sfc3hnoxBv09qkpgGFCen/T+/06f21nLZOyJkUijxxRmoI1IwNLWpoRljozp4v5KCIU1hvzTpoyucu4OZdfjnvlKmruuIOkadNI3nefLs12xqSPiYS4OpvdCTmZwziKx9P2/vsovx+xG5nM2nw0sCTajlOj2W0IhUL87eu/YQ+k8Iuy18CZCsf9Ca5fDZd9CAf/DPKnDphAiBcqKiJMzpoc13wUq8RFvLHGZ4ynydtEk6cp4XmF6x75q8xw1G7OYfuoUfiqKsl15XZxNId7M3vWr8OaldWjhaRYLIy88w6sGRk0PfUU0DOrOVwMz97Y1i+h4CwuhkAA36ZNkXXafDSwaKGg+d7x4LL/4LaWcLU3SFJKgVHyev/LIGPgm7kH6uooOfgQWl5/Peb2KVlT2NC8gWAo2GNb9xIXbe+9x4a58/DX1PbYN9rZnAid/k7a/e1GH4XNm8FqxV7YNXLJMaooEpYazmoOm5zAMB85p0yJafqxpqWRPGsW7hUrgJ4JbGFNwdrQklDkUWROZlhquNyFUopgu44+Gki0UNB8rwiFQvxr1T9ICSRzQe16OOp34By8p8y6++4jUFdHy5tvxtw+OWsy7oCbyvbKHtvCmkLYfNTx+RcEm5tpeuaZHvuGw1ITFQpdO65VYh8xArF1tSbbi0YZlVKTsmnwNNDqa8UX8pHrykUFg3hLSnqYjqJxzZiBv7KSQENDjwS2Fm8LTp+Cjs6EIo/COMYbYanhchfK64VAQAuFAUQLBc33ins+eRmfdSM3tjVhHzUHpp/Zr+P9W7fS+u9/J5Q97FmzhuYXX8KSkkLnF0sJeb099glHIMXyK4RNNmHzUdjB2rxoESG3u8u+I1JG4LK5EnY2d+nNXFnZJfIojH3UKJTHQ6EnCW/QS0VrBbCtdadyu7s4mbvjmmFEbbm/NbWF5IJIAlurr5WsdmO//piPrKkp2AoKIglsoXZjEOsu2nVtOKKFguZ7gy8Q4Jn1D5Hjd3JKc41Rx6iXGv7d8dfWsvH8C6i64UaaX3yx132VUtTccSfWjAwKf/c7lMdD55c9s5cnZE7AIpZehUJWktF8xrt+Pc5JEwm2tNDy2mtd9rWIhXHp4yhvKe8xTiwimoIrH19lJY7RsYSCEZaabwQKsa7RcMbnunLxrA9HHk2Je46kPfcEqxX3im8BI4s72nyU22589v0RCgDOCcV4y43rDAsFrSkMHFooaL433PHxIgK2rfy8qRrbzPOgqGcYaDyCLS1svuRSAo2NJE2fTs3v/xC5Mcai/f336fziC3Kv+QlpRxyOOBx0fPxRj/2SbEmMTR8bueFG0+BuIMOZgd1iJ9DQQLCxkczTTydpr71oXPh4jwJ04zPGJ2w+CmsKOaQSbGjAPmp0j33C0UiZjT4A1jauBYwSF95160EE58QJcc9hcblwTpmM+1tDKBQkF3RxNI/0GAmB/RUKjvHF+MqMWk/BNlMopGihMFBooaD5XtDp9/JS2aOM9tk4OiBw5C0JHxvq7GTzFVfiKy9n9N//xuh/3I8lNZWqG27oYcYBUD4fNXfdhWPCBLLOOguLy0XyfvvR/vGSmONPzpocU1OITlzzrt+WE5B94YX4Kipo/6irkCnOKGZrx1Y6/Z09xupOvbseh8WBq9aMAjK1gmjsRca6tHpjvLDgykvOw7t+PY6xY7tkLcfCNWMGnhUrUcEghSmFNHmb8Aa9tPhaKOxH3aNoHMXjCbW3E6itI9ShzUcDjRYKmu8Fv3v/CUK2Om5u3Ir1kJshNbEbkfL5qPzpdbi//ZaRd99Nyrx52PLyKPrTXfhKy6j+wx96HNP49DP4N26i4Oc3R5y3qQcdiK+sDF9lVY/9p2RNoaq9inZfe5f10SUuIkJh0iTSjz0GW0EBjQsf77J/2Nlc3tq3CanWXWuUzK4y5tO9VhEYT/rW3Fyctc2A4fdw2Vyk2FPwrF/Xqz8hjGvvGYQ6OvCVlW3rq9BRQ6u3ldwOK5KcHLcPQzycUV3YtPlo4BkSoSAiFSKyUkS+EZFl5rpsEfmviGww/8bv4q3R9EEoFOKbrRXc++mrLHjlD7xV9RhTvYqDk4tgv8sTGkOFQmz55a/o+PhjCm+9hfRjjo5sS5k3j5zLLqPlxZdoeWNxZH2gqYn6f/yDlAMPJPXgg7ftf5DxvmNJzyqikXIXzV3zFaI1Bc+GDVizs7Hl5iJ2O1nnnUvn55/jWbs2sn8kLDUBZ3NdZ53pZN4MENPRDOAoKkK21iMInqCHPFceoc5O/Js24+wl8iiMa8YMANwrVnRJYDMczQp7Xl7C2cyRORVvC0uNmI+0UBgwhlJTOEwpNVMpNdtc/gXwnlJqEvCeuazRJExteytnv/BbDnjsDGYsnMv57/yQRzb8H8taniMzFOSW+hosx94Btr7r7iulqPn9H2h9803ybryBrDN7RinlXfMTXPvuS/Utt+CrqACg/m9/J9TZScHPb+6yr2P8OOxFRbR/FF8orG/sakKKLnHhXb8B56Rt/aCzzjwTcblofPyJyLoxaWOwirVPZ3MwFGRNwxomZk7EV1mJJTkZa1bsZzD7qFEEqqoizu5cVy7ekhJQiqQp8Z3MkeseNxZLejrub1d0SWBr8baQ0Rrst+kIDHOTJTkZX1m51hQGgeFkPjoJCOvDjwMnD91UNLsiP3njflZ3vkJAuRmTdABH5V/Fb2bez0d73sRHtZXsNfYwmHRU3ONVMIhnzRoan36ayquupumZZ8i++GJyLrkk5v5is1F0z92I3U7lDTfg+e47mp57jqyzzuxyAwcjeznl4IPo+PxzlM/XZVthSiFpjrQufgV/0E+br42cpBxUKIS3pKSLucaakUHmKafQungxgTqjcqndamd02ug+nc0lzSW0+duYVTDLyFEwu63Fwj5qFP6tW8m1G0IhLzkPzzrDt5CI+UgsFlzTp+P+9tsuCWytvlZSW/3bJRREBEex4WyO+BT6aYLSxGeoah8p4B0RUcCDSqmHgAKl1FZzezVQEOtAEbkMuAxgzJgxO2Ouml2AbzY3s6plCbmp4/jowjeMlY1l8OaNUPo+jNwXjr+nx3GetWtpe/c93F99hfvbbwl1dADG02jOpZeSd8P1vZo37CNGMOL//ZHKq65m43nnY0lOJveaa2Lum3rQQTQ/u4jOr74i5YADIuvD5S7WNW2LQAqHo2a7svFXVaE6O3FOmthlvOwLzqfp2WdpevZZ8q69FjC7sPUhFJbVGKGxswtm4658GHsv/0f2UUUQDDLWm8p6zMij9RuQ5OS4JqfuuGbMoP6f/8TpDZHhzGBr+1bavK24mvvvZA7jnFBMx9Ivce4xFXE6kV2069pwZKg0hQOVUvsCxwFXi8jB0RuVkRkUMztIKfWQUmq2Ump2Xl7i6fGa3ZdQSPHr1z/C6trEmdOOh4APPr4H/jEXNn9p1DW65F3I7OpM7Vy+nIozz6L+/vsJNDSQfuIPGfmnu5jw7rtM/N+H5N94Q0L27rTDDyf7wgsIdXaSe+WV2OKYYlL23x/s9rgmpA1NGwgpI8w0kriWlBNJWkvq9mTuGDeO1MMOo+nZRYQ8Rknq4sxiNrduxh/yx53v8prlFKUWUZhSaOQoxIg8ipzDvPGPbjOa7eS6cvGuW4dz0kQkwRwP14y9IRTCvWo1BckFlDSXkORVWL2B7RYKjvHFBLZuJVBbp01HA8yQaApKqSrzb62IvALsB9SIyAil1FYRGQH0LPCi0cTg+WWbWd/+GUmpcEJyITx4MNStgT1ONBLU0nuWVfaWlbP5qquxFxUx9sknsHUr6tZf8n/2M1IOPIiUeXPj7mNJSSF51iw6Pv4Ybr6py7YpWVPoDHRS1V7F6LTRNLqNukfZSdl4138OgGPipB5jZi+4kE3vv0/L66+TdeaZFGcUE1ABNrdujkQjRaOUYnnNcg4sOpBgYyPK7Y6ZoxAmrA0UtgrkQZ4pFNKOPjruMd1J2tvMbF7xLYXFhXxd8zXZkWzm7Xuwc5hd2DwrV2LVQmFA2emagoikiEha+D1wNLAKeB240NztQuC12CNoNNto7vRx53/WkpW3hslJeYxbdCH42uGcRXDWkzEFQqC+ns2XXYbYbIx+6MEdFggAYreTetCBiLX3NiSpBx2Ed8MG/Fu3dlnfvdxFdIkL7/r12IuKIv2To0meMwfntD1ofPwJlFJ9FsYrby2n0dNo+hPCkUfxNQV7YSFYLOQ0GQX78jptBFtaEvInhLFlZWEfO8bwKyQX0OZvI6sfHddiEQlLrajQmsIAMxTmowJgiYh8CywF3lRK/Qe4AzhKRDYAR5rLGk2v3P3OOtoCDbitpRxdXQrFh8FVn8OU42LuH+rsZPOVVxGor2f0Px+IGZ8/mKQcdCAA7Uu6JrJNyJyAIJEIpOgKqd4NG+LehEWEnAsvxFdaSucXXzA+w3iCjicUltcsB2BWwaxtfRR6+QzEbsdeWEhmo1G3KW+L4XNJJBw1GteMGYZQcBlCIKvNWG/bThOwfcwYMAWwFgoDy04XCkqpMqXUDPO1p1LqD+b6BqXUEUqpSUqpI5VSjTt7bppdi1VVLTz9xSaOnGIUXDvaXgBnPm70R4iBCgapuvFneFavpujP9+CaPn1nThcwks9shYV0dPMrJNuTGZs+dpum4DYqi7qUDW95RY9opmjSjj4aMX0VyfZkClMKexUKua5cxqSN2dZHoSi+pgCGCSmvSfG3w/9GVpVxN+/u3+gL194zCNbVM6rTyICOmI+2UyhYHI6Iv8Ois5kHlOEUkqrRJEwopPjta6uYlNyJ272YiYEQ43/0IiTF7vWrlKLmD3+g/YMPKPjNr0k7/PCdPGMDESH1oAPp+OwzlL+rM3hS1qSIUAgnrnnLKyAQ6NVcY3G5cM2eRccnnwDEbc2plGJZ9TJmFcxCRPBt3ow1L7fPUhXhXIVDRx+KZ906bAUFWDMz+3Xd4YqpBRuNshpZ7QpSUvqdzRyNY4JRd8mq6x4NKFooaHZJXvqqktWbavl7+r18bVMcPfk0yIwdWqlCIRoe/hdNzzxLziU/JvtHP9rJs+1KykEHEWpvx/3NN13WT86azOa2zXT6OyMlLrbVPIqvKQCkzp+Pd906/LW1FGcUU9FaEYlkCrOlYws1nTWRftD+yiocvTiZw9hHFRGoqyPk8ZiNdfqnJQAkTZmCOBxkbDAK4mW1g307ncxhnKazWZuPBhbdo1kzbPH6/dS2e7BZuv5MPf4Qd731HY9m/IvlgY0oyeLo6Rd22Sfk89H5xRe0vfsebe+/R7CunvQfHEfeDTfszEuIScrcuWCz0f7xEpLnzImsn5I1BYViQ/MGGjwNjEwdifebDWCz4Rw3rvcx58+Hu++h87PPGD91PO6Am+qOakambnO0L6velp8A4N+8GdesvivFhs00vo2b8JaVkWr6RfqDOBwkTZtGaE0ZjIWcdsE+KmYqUsKEu7Bp89HAooWCZliyrm4LZ79+MV6/0LnxclD2Lttvti1ifnAJ/5qwPxMcyUzInIBSira336btnXdo/99HhDo6sCQnk3LwwaQdcQTpxx6TcGz9YGJNSyN55kzaP/6Y/Buuj6yfnL0tAqnR08j03OlGD4Xx4/tMznJOmYI1J4f2JZ9QvL9RkqOspayLUFhes5wMZ4bxWfn9+KuryYjRR6E74bDUjiVLwO/HmUB5i1i4ZsygadEisk9IJ7u9ebtzFMKEw1J1SOrAooWCZtjxXU0V5y5egLLUYHUpzh/3O67vcEa2iwqS1bae+n1+xPLmT7h8slHgrvm556m+9VasOTmk/+A40o48kuQDDsDidMY71ZCRctBB1P3lL/hrayNhmSNTRpJqT2Vtw1qaPE1mjsLHuGbO7HM8sVhImTePjk8/ZfzvjLpLZc1lHFi07al+ec1y9s3fF4tY8G2thFAIe1ECQsHcp+3994HEylvEwjVjbxoff5y9m0eS2da03eGoYZyTJmPNy8UxcWLfO2sSZugfmzSaKFZsqeS8Ny7AYqnmnzXVXEgGryYFWJ6fTXbhOLILx5E1YgLMu4b3Jh+EQnH02KNRwSANjz5K0vTpTProf4y4/XZSDzlkWAoEgNSDDwKgY8knkXXhchdf1nxJUAXJU6n4t2xJ+CacMn8ewYYGUjbWkenM7BKBVNtZy6a2TVH+BDPyKAFNwZaXizgcuL/+2jBlmX2S+0u4YuohlanYgmqHNQVragqTP/6YtEMP3aFxNF3RQkEzbFi+eTM/fvM8LNYa/l7XxAEnPMBPz/uA6bnTucXSTOUP74YfLTJeR/+e/256n3Hp45iYOZG2/76Lf9Mmcn784z4TyIYDzqlTseXl0fbuu13WT8qaFKlyWlBjFM5LWCjMmwdAxyefUJxR3KVa6lc1XwHb/AmRHIUE6heJxWKErYZCOIuLt7vOkG3kSKy5uey7NmAs76BQ0AwOWihohgVfbNzINf85G2Wr597WEHPPWwx7nYbdaueug+8C4OaPbsYfNMI4G9wNfFnzJUePM8otNDzyCPYxY0g76sghu4b+ICJknHoq7R98gG/jxsj6cGYzQFaVEb7ZV+RRGHt+Ps7Jk2n/5JMerTmX1Swj2ZbMlGzDH+CvrAS7HVtBYs7esF9he01HYFyza8YMvGvWAFooDFe0UNBsF26/j++qa6mo79jh13++W8/P3zkDv62ZP/uzOPDHH8LImZFzjUobxa3zbmVl/Uru+/o+AN7f/D4hFeLosUfT+eWXeFauJOfii3YJLSFM1rk/Qmw2Gh/f1kEtfNMGSN5UjyU5GfvInqU64pEyfz7uZcuZmDSaZm9zJDN6ec1y9snfJxLJ5avcjH3kiIQ/r3ApjO0JR43GZdZBAi0Uhiva0azpFy1uH3e8/xwfbL2fgMXDiVXjKPQmJ3SsoLATwIkfh/hxEqDT6uGxokY6bCHuStqbg89fCLaefoCjxx3NWdVnsXD1QuYUzuGdincYmz7WiO1/5Aqs2dlknHzywF7sIGPPzyf9xB/S/PIr5F5zDbasLCZlbtMKbBVVWPpRjRQModD42GNM2mhoVGXNZUimUNJcwvHFxwNGuK77629Imjo14XHDZqb+ZjJ3J5zEBtufzawZXLRQ0CTElmY3T731JEuaH6U8uYNxyk8gJLw5agP31bUyyxu/VHM0IYudkMVB0Opkvd3OjRkhOizwp5Enc+hRv4eoUtXesnLEIjjMGP2b5tzE17Vf8+slv6bN18bFe12Md8MGOv73EbnXXoMlKWkwLn1QyVmwgJaXXjZ6Ilx1Fcn2ZEanjWZLWxXBknKSjzyiX+Mlz56FOJ3krqyEsUZYaouvBSDiZG5+7nkC1dVk/eH3iY+73344xo+PVDzdXpL2mg4iWNPTh20QwPcdLRR2Aeo72nH7VI8kLoBAUOENhPD4g3j9QfzuVppbN7GlvYQxSWNJsuzYjVKFgmxa8Sbf+v7NOxkKV5LiGhnNRYfdRFPhnlz67uX8xL6Few+7l/lF8xMe9/Otn3P9B9eTbE/miSP+0cVsEmxro+5vf6PpqaexJCczZuFCXHvtidPq5E+H/ImzF59NUAU5auxRNP7pMcTlIuucc3boOocK56RJpBxyME1PPU3Oj3+MxelkavZUHM2dBJtqeq15FAtLUhLJs2fj/+IbXBNclLeUU9FagdPqZM+cPQl1dlL/4IMkz5kTcUwngmv6dCa89e/+Xl4PrKkpOCdOJE67FM0wQAuFBAkEQ3gDob533JFzhBSbGjopraqmZeMKAjVroPU7nixYQ6clxMxOC7M6LEzvtOBUxhO1Az/pdJJk6eTLFMV/UpP5wpVEUASbUszweJnr8XCA28OeXl9CX7gCaq1W1jnsrHI6eTY9lVaXlRPS9uTGw+8gO8sIScwHHjv2Ma747xX85P2fcPfBd3PE2L6fbN8ofYPffvJbxmWM44EjH4g0dFdK0bp4MTV33UWwvoHM00+n45NP2HzJJYx96kmcEydSnFHM/zvw//FR1UdM8GVSungxWeecE7exza5AzkUXs2nBAlpee42sM8/kp/v+lIb2D4A7t8uxmzJ/PrV33cXeoamUtZTR7G1m77y9cVgd1D/9OMH6evLu+2tCDYQGg7wbrkf5EtMsNTsfMZqc7ZrMnj1bLVu2bFDPsbXFzZ3vPs6nzY9gDwnFLfmMb83HFdwWlmcnQLJ4cWG8ksVL0OKmw+4j128hOZTYP5+VEGMtNYySegACwKWFBXyb5OTggIulNg9tonAomB1I4sCAC6fY+MDh4wtLK34UBZYUDkmewh4p4/jOU8XX7gpKfTUoIMXiZK+kUWRaU0gSO06LHafYSBI7drFRE2im1FdLmbeW1pA7Mq9ZmdP4xUG/Y2q2YYNWoRBt77yDv6qKjFNPpTPFylXvXsWq+lXcPv92fjjhhzGvTynFwysf5m9f/439C/fnL4f9hTRHGgDeDRuovu12Or/8kqTp0yn87W9xTd8L38aNVJx3HoIw9pmnu5R5rrnzLhqfeIIJb7/da/ew4Y5SiorTTifkdlP85mLEYqFh4UJq77iTSZ9+gi07u1/jedatp/ykk/hkwUyeGFdFi6+Fy/a+jCsmXEDJkUfhmrE3Yx56aJCuRrOLEPempDWFOKwoq2LJfx/lc/+rfJ3mY0rQR7IK8XWem1W5FRzW6ebUtnbmuj1YgTaLnS+TU/kiKYkvk+yU2MB8mCc9JIwKWigKWhgVsjAuYOUAvw1rj+/FSjBjfxpG7knmmL25p+EzlpW9zt/bfsgevlzsUydTUqB417+S9yrf59POGsDoznXGuHP4QfEP2NMxDv+mzYTa2zhl7jSsGRk0eZr4ovoLPt/yOV/Xfk25rxp3wI076CYQCkTO7rA4mJQ1iaNGzWZK9hSmZk9lctZkUuxGJUulFB2ffErtn+/B+50RVlh3/z/IOvtsHjj/j1y/4jZ+veTXNHoamZw1GV/QhzfojbyW1yxncdliTig+gdvm3YbdakcFAtT99T4aHnsMS0oKhbfeSuYZp0eiYhxjxzLmkUfYdP4FbFpwEWOfeRp7QQHB1laan3+e9GOP3aUFAhihmtkXX8yWn/2M9g//R9rhh+HdsAFrTk6/BQIYIazWvFwmrXfTNKIJMPwJjY8tJNTSQt5PfzrQl6DZjfheagpry77io5WvMNE+kkxr17opPncbgTWLabSu5J7cNFotFi5yTeHSqZfjHDuLCn81L5e9zuvl/6bJ20xhcgGFyYWsalhNQAWwW+wc317Msf/rIH1zE3XzJvP1/HxWuxrZ2LqR2k6jy+j8ovncedCdZDhjl3p+o/QNfvXxL/l/X09hwtvfgcUCIcN8ZUlJwTl1Kh3j8wmkOMlpDBDYtBnfpk0Em5q6jOOcNAnXvvuSPGtfXPvui72oqIvZwB/y4wl48AQ8ZCVlxfRbALhXrqT2nj/T+fnn2EeOJO+n15I0bRr1Dz1M65tvInY7aaefyl/32MhbHUtjjiEIl0y/hGv2uQYRIdjSQtX1N9Dx6adknHIK+Tf9LO5N0L1yJZsuXICtsJCxTz1J80svUXfPnxn/8kskTZsW85hdCeX3U3L0MThGjWLsk09QfsaZWFJTGPvYY9s13paf/4LGD97lR1d5sFrsfHTsYrYceyIpBx7IqPv+OsCz1+yCxNUUvpdC4Z8Lr2HphvepyBdsKUH29nqZ4fWxt9dLfiDI73Py+SjFznzPCK53H4pjydd4Vq1CXC7SDj+c9BOOx3nA/vyv9hNe3vAyrd5W5hTMZv7WNHKe+xDvsuVYs7Jw7b037Z98AoEAKQceSNaPzsEybw5vVLzJnV/eyYiUEfz1sL8yKaurM3F1/Wou+Pf53PBxBvt+XE3WBeeTf+ONeEtK8K5Zg+e7NXi++w7PunUojwfbiEIcY8biGDMGx9ixOMaOQVwuPCtW0PnV17i//ppQu9HVxJqXi3N8MfbRo3CMHoNjzGjso0djHzUKsVoJuT0or8f463ETbG+n+YUXafvPf7BmZZF75RVknn02lqisVl9FBfUPPUzL668bdfqPO5jgeSfiGFmEw+ogyZqEw+og2Z4cMRf5KirYfMWV+KqqGHHLb8k8/fQ+v7eOpUvZfOllOIqLCdbX45w0iTGPPtLv73+40vDYQmrvvJNxzz/HxgsXkHnG6RT+6lfbNVbLG2+w5aab+fkCK+l7z+SeVTNoXLiQ4tdfMx29mu85WihEs/npx2m/3ej26U2ysjnfQklugI0FQnUm7FVp5diNGaRsMmz7SXvvTdoRR+DfuoW2t/5DsKUFa0YGacceS8YJxxNsa6P+nw/iWbECW34+OT++mMwzzsCSnIy/ppbmF18wwgBra7GNGEHWWWdSecSeXP/1b+nwd3D7/Ns5ZtwxgJGpe/brZ3Lmmy0cuLSD7AULyP/5zTGdgioYRAWDXW7QsVDBIN6SEjqXL8fz7Qp8mzbhq9xMsK4+oc9LkpPJWbCA7Isv6rUipa+yioaHH6b55ZcRIPOM08m5/HLs3bJmOz7/nMqfXoeIMOpv93UpH90X7R99xOarfwJ+P6Mf+Rep8xOPeBruBNvbKTn0MBzFxXhWrGDE729PSFjGIlBfz4YDD+Klw5MoOP1sDrr+WdKPPZaRd+outxpAC4WuhNxuvOvX41m7Du+6tXjWrcezdg2qo9PYQQTXrH1JP/po0o46CvuIEZFjlc9H+6ef0rr4Tdreew/lNhyy9qIici69lIxTT4l5k1Z+P20ffEDTs8/S+dnniMuF85QTuGvCWpYE1nDxXhdz1cyruPzty9jvia847JsAOZdeQt4NNwxalEiosxPf5kr8lZvxV1ailMKS5MLiSkLCf51JOCdP6ld0j3/LFuoffIjml15CLBYyzzqLnEsvwZ6fT9OiRVTf/nsc48cx+oHt65Hc9uGHuJd/Rd4N1w9ZBM1gUfOnP9H4yKMAjHv+uS4ZwP2l7JRTCSY7SZ44mdaXXmbCW//e6T2pNcMWLRT6QimFv6oKX3k5SWaxsr4IdXbS9sEHiAhpRx2F2O19HgPgWb+exkceoWXxmyBC+QGjuW+PTQSK8jn1pWoOW6nIueJy8n760136puerrKLhwX/S/PIriM1G8qx96fj0M1IOOZiie+7RdfBj4K+upuTIoyAQYMryZTvUrrL27rtpWPg4iJB52qmMuPXWgZuoZldHC4XhiL+qioaFj9P8wgsoj4eqXKGoXpF79dXk/uTqXVogROPbvJn6B/5Jy+uvk33eeeTf9LNdqkbRzmbrLbfiWbmS8S+/tEPjdHz2GZsuuhhxOpnwzts9zHia7zVaKAxnAk1NND35FE0vvkD2j84l94rLh3pKg0LI5+vT/6ExfEAohdh2LGI85PVSctjhZJ52Gvk3Dn0bUs2wQgsFjeb7SKijA3G5hkUbUs2wQievaTTfR3bEJ6H5fqIfHzQajUYTQQsFjUaj0UTQQkGj0Wg0EbRQ0Gg0Gk0ELRQ0Go1GE2GXDkkVkTpgYx+75QKJFfnZvdDX/f3j+3rt+rr7T71S6thYG3ZpoZAIIrJMKTV7qOexs9HX/f3j+3rt+roHFm0+0mg0Gk0ELRQ0Go1GE+H7IBS+r81o9XV///i+Xru+7gFkt/cpaDQajSZxvg+agkaj0WgSZLcWCiJyrIisE5ESEfnFUM9nsBCRR0WkVkRWRa3LFpH/isgG82/irdN2EURktIh8ICLfichqEfmpuX63vnYRSRKRpSLyrXndvzPXjxeRL8zf+3MislvWKRcRq4h8LSKLzeXd/rpFpEJEVorINyKyzFw3KL/z3VYoiIgVuB84DpgGnCMi04Z2VoPGQqB7zPEvgPeUUpOA98zl3Y0AcKNSahpwAHC1+R3v7tfuBQ5XSs0AZgLHisgBwJ3AX5RSE4Em4MdDN8VB5afAmqjl78t1H6aUmhkVhjoov/PdVigA+wElSqkypZQPWAScNMRzGhSUUh8Bjd1WnwQ8br5/HDh5Z85pZ6CU2qqU+sp834ZxoyhiN792ZdBuLtrNlwIOB1401+921w0gIqOA44F/mcvC9+C64zAov/PdWSgUAZujlivNdd8XCpRSW8331cBu3YtRRMYB+wBf8D24dtOE8g1QC/wXKAWalVIBc5fd9fd+L3AzEDKXc/h+XLcC3hGR5SJymbluUH7nusnO9wCllBKR3TbMTERSgZeA65RSrdG9rXfXa1dKBYGZIpIJvAJMHdoZDT4icgJQq5RaLiKHDvF0djYHKqWqRCQf+K+IrI3eOJC/891ZU6gCRkctjzLXfV+oEZERAObf2iGez6AgInYMgfC0Uuplc/X34toBlFLNwAfAXCBTRMIPervj730+cKKIVGCYgw8H/sruf90oparMv7UYDwH7MUi/891ZKHwJTDIjExzA2cDrQzynncnrwIXm+wuB14ZwLoOCaU9+BFijlPpz1Kbd+tpFJM/UEBARF3AUhj/lA+B0c7fd7rqVUr9USo1SSo3D+H9+Xyl1Lrv5dYtIioikhd8DRwOrGKTf+W6dvCYiP8CwQVqBR5VSfxjaGQ0OIvIscChG1cQa4BbgVeB5YAxGJdkzlVLdndG7NCJyIPAxsJJtNuZfYfgVdttrF5G9MRyLVowHu+eVUreJSDHGE3Q28DVwnlLKO3QzHTxM89HPlFIn7O7XbV7fK+aiDXhGKfUHEclhEH7nu7VQ0Gg0Gk3/2J3NRxqNRqPpJ1ooaDQajSaCFgoajUajiaCFgkaj0WgiaKGg0Wg0mghaKGh2OiKiROSeqOWficitAzT2QhE5ve89d/g8Z4jIGhH5IMa2SSKyWERKzbIEH4jIwYM9p3iIyMnRxSBF5DYROXKo5qMZ3mihoBkKvMCpIpI71BOJJiorNhF+DFyqlDqs2xhJwJvAQ0qpCUqpWcA1QPHAzbQnZlXgeJyMUSkYAKXUb5VS7w7mfDS7LlooaIaCAEYrweu7b+j+pC8i7ebfQ0XkfyLymoiUicgdInKu2VdgpYhMiBrmSBFZJiLrzXo54QJyfxKRL0VkhYhcHjXuxyLyOvBdjPmcY46/SkTuNNf9FjgQeERE/tTtkHOBz5RSkex5pdQqpdRC89gUMfpfLDV7Apxkrl8gIi+LyH/M+vh3Rc3haBH5TES+EpEXzFpP4Rr7d4rIV8AZInKpeX3fishLIpIsIvOAE4E/iVGLf0L0ZywiR5jzWGnOyxk19u/Mc64Ukanm+kPMcb4xj0vr68vW7FpooaAZKu4HzhWRjH4cMwO4AtgDOB+YrJTaD6OM8jVR+43DqA1zPPBP8+n9x0CLUmoOMAe4VETGm/vvC/xUKTU5+mQiMhKjVv/hGH0L5ojIyUqp24BlwLlKqZu6zXFP4KteruHXGOUZ9gMOw7hZp5jbZgJnAdOBs8RoIpQL/AY4Uim1r3neG6LGa1BK7auUWgS8rJSaY/ZZWAP8WCn1KUY5hJvMWvylUdeXhNGL4yyl1HSMbNkro8auN8/5APAzc93PgKuVUjOBgwB3L9eq2QXRQkEzJCilWoEngGv7cdiXZg8FL0ap6HfM9SsxBEGY55VSIaXUBqAMo4Lo0cAFYpSb/gKj5PIkc/+lSqnyGOebA3yolKozSzM/DfTLNyAir5haRrhY39HAL8x5fAgkYZQpAKNhSotSyoOhtYzFaB40DfjEPOZCc32Y56Le72VqPSsxNJY9+5jeFKBcKbXeXH682/WF57ycbZ/vJ8CfReRaIDOqZLVmN0GXztYMJfdiPFU/FrUugPmwIiIWILq1YnQ9m1DUcoiuv+XutVsUIMA1Sqm3ozeYNXQ6tmfycVhN1I1VKXWKiMwG7g6fEjhNKbWu2zz2p+v1BTGuSYD/KqXOiXO+6LkvBE5WSn0rIgsw6mHtCOH5hOeCUuoOEXkT+AGGoDpGKbU23gCaXQ+tKWiGDLN41/N0bZ9YAcwy35+I0VWsv5whIhbTz1AMrAPeBq4Uo9Q2IjI5ymwTj6XAISKSazpyzwH+18cxzwDzReTEqHXJUe/fBq4RMZo+iMg+fYz3uTneRHP/FBGZHGffNGCreY3nRq1vM7d1Zx0wLjw2hkmu1+sTkQlKqZVKqTsxKhHv9n0cvm9ooaAZau7BqO4a5mGMG/G3GD0CtucpfhPGDf0t4ArTHPMvDJPMVyKyCniQPjRls6vVLzBKM38LLFdK9VqeWCnlBk4ArjAd4p9h+AR+b+5yO4agWyEiq83l3sarAxYAz4rICuAz4t+I/w/DNPYJEP30vgi4yXQMRxzy5udyEfCCaXIKAf/sbT7AdaY5bAXgx/iMNbsRukqqRqPRaCJoTUGj0Wg0EbRQ0Gg0Gk0ELRQ0Go1GE0ELBY1Go9FE0EJBo9FoNBG0UNBoNBpNBC0UNBqNRhNBCwWNRqPRRPj/ytmJl+peabYAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], "source": [ - "num_generations = 250\n", + "num_generations = 50\n", "num_rollouts = 20\n", - "print_every_k_gens = 20\n", + "print_every_k_gens = 5\n", "\n", + "rng = jax.random.PRNGKey(0)\n", "es_logging = ESLog(param_reshaper.total_params,\n", " num_generations,\n", " top_k=5,\n", @@ -364,11 +508,117 @@ "es_logging.plot(log, \"CartPole Augmented Random Search\")" ] }, + { + "cell_type": "markdown", + "id": "50ba4ab2", + "metadata": {}, + "source": [ + "# More Minimalism (no `es_params`, `fit_shaper` or `param_reshaper`)\n", + "\n", + "We also provide utilities that abstract away all the details if you are only interested in a default implementation or want to avoid 10 additional lines of boilerplate code :)\n", + "\n", + "This means that you can directly provide the placeholder parameters and fitness shaping arguments at the time of strategy instantiation. Furthermore, if you don't explicitly provide `es_params` at the time of `initialize`, `ask`, `tell` the strategy will use a set of default parameters:" + ] + }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "e1fab05b", "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ParameterReshaper: 4610 parameters detected for optimization.\n" + ] + } + ], + "source": [ + "strategy = ARS(popsize=100,\n", + " pholder_params=policy_params,\n", + " elite_ratio=0.1,\n", + " opt_name=\"sgd\",\n", + " maximize=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "07ab2a62", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generation: 0 Performance: 22.875\n", + "Generation: 5 Performance: 26.25\n", + "Generation: 10 Performance: 27.8125\n", + "Generation: 15 Performance: 31.3125\n", + "Generation: 20 Performance: 53.0\n", + "Generation: 25 Performance: 99.0625\n", + "Generation: 30 Performance: 115.8125\n", + "Generation: 35 Performance: 130.125\n", + "Generation: 40 Performance: 192.9375\n", + "Generation: 45 Performance: 200.0\n" + ] + }, + { + "data": { + "text/plain": [ + "(
,\n", + " )" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAADgCAYAAADsbXoVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAB3DElEQVR4nO2dd3hb5fmw70fT8t524kxnEggJJAGSsDeFsmdZgbIplFHo/BUKbT+g0FJaSoECYYe9QimUVQgrJIwMsryS2In3trX1fn+cI0W2JVtO7NgJ731duqyz3vMeST7PebYopdBoNBqNBsAy1BPQaDQazfBBCwWNRqPRRNBCQaPRaDQRtFDQaDQaTQQtFDQajUYTQQsFjUaj0UTQQkGz3YjIoSJSOdTz2NURESUiE4fBPBaIyJKhnsfOQkRuFZGnhnoeww0tFHYBRORHIrJMRNpFZKuIvCUiB+7AeF1uQubNPWSO3yYi60TkooGZfZ9zudWcz/4743wDzWDfSEXkQxHxmN9NvYi8LCIjBut8OwsR+bGIrDV/bzUi8m8RSRvqeWm0UBj2iMgNwL3AH4ECYAzwD+Ck7RjL1svmLUqpVCAd+DnwsIhM6/eE+zcfAS4AGs2/mtj8xPxuJgKpwN1DPJ8dQkQOwfg9n6OUSgP2AJ4bhPOIiOh7XD/RH9gwRkQygNuAq5VSLyulOpRSfqXUG0qpm8x99hORz0Sk2dQi/i4ijqgxlIhcLSIbgA0i8pG56Vvz6fOs6HMqg1eBJmCaiDhF5F4R2WK+7hURZ5z5jhSRl0SkTkTKReTaPi7xIGAEcC1wdrd5d1HtRWSceS02c3m8iHxkPmm+KyL3h/eP2vciEdksIk0icoWIzBGRFeZn9fduc79YRNaY+74tImO7fYZXiMgG89j7zRvOHsA/gbnmZ9ls7u8UkbtFZJP5FPxPEXFFjXeT+V1tEZGL+/iMIiilmoFXgZlRY11kzrtNRMpE5PKobYeKSKWI3CgiteY5L4raniMir4tIq4gsBSZ0+0zmiciXItJi/p0Xte1DEfm9iHxqXvsb5nhPm+N9KSLj4lzKHOAzpdTX5nU1KqUeV0q19fX5iUiWiCw2f2NN5vtR3eb1BxH5BOgEikVkTxH5r4g0muP9KmouDhF5wvz8VovI7ES/j90WpZR+DdMXcCwQAGy97DMLOACwAeOANcB1UdsV8F8gG3BFrZsYtc+hQKX53gKcAviBKRhC6XMgH8gDPgVuj3PccuC3gAMoBsqAY3qZ+yPA84AdaABOi9p2K/BU1PI4c942c/kzjCdmB3Ag0BreP2rffwJJwNGAB+OGmg8UAbXAIeb+JwElGE+sNuA3wKfdPsPFQCaGplYHHGtuWwAs6XZdfwFeNz/zNOAN4P9Ffac1wF5ACvBM9++j21gfApeY73OAd4HXorYfj3EzF+AQjBvhvlHfT8D8Du3AD8ztWeb2Rebnn2LOpyp8Lebcm4Dzzc/kHHM5J2peJea5M4DvgPXAkeb+TwCPxbmmgwA38DtgPuDsx+eXA5wGJJvbXgBe7fZ5bQL2NOeRBmwFbjR/C2nA/lG/MY/5uViB/wd8PtT/90P9GvIJ6FcvXw6cC1T385jrgFeilhVweLd9YgmFENCMYcr5Bjjb3FYK/CBq32OAiqjjwkJhf2BTt/P8spcbQzLGjfxkc/lBut7sbiWOUMC4MQeA5KjtT9FTKBRFbW8AzopafglTeAJvAT+O2mbBuHmOjfq8Doza/jzwC/P9AqKEAsbNuQOYELVuLlBuvn8UuCNq2+Tu30e3z+lDcy4t5n7fAGN6+f5fBX4a9f24iXqowBCGB5g3QT8wNWrbH9kmFM4HlnYb+zNgQdS8fh217R7grajlHwLf9DLP4zBu9s1AO/Bnc069fn4xxpkJNHX7vG6LWj4H+DrOsbcC70YtTwPc2/v/uru8erMxa4aeBiBXRGxKqUCsHURkMsY/1GyMG60N44k9ms0JnGuLUmpUjPUjgY1RyxvNdd0ZC4wMm1BMrMDHcc53CsaN/d/m8tPAuyKSp5Sq62OuI4FGpVRn1LrNwOhu+9VEvXfHWE6NmvtfReSeqO2CoVGEr706altn1LHdycP4HpaLSPRY1qi5R38/0Z9tPK5VSv1LRKZjaCyjMJ6GEZHjgFswhIvFPPfKqGMbuv12wnPPw/itRP82oufS/XsPby+KWk708+2BUuot4C0xbP6HYTzxrwNeoZfPT0SSMTSJY4Esc3uaiFiVUkFzOfqaRmM82MSj+/ea1Nv/2/cB7VMY3nwGeIGTe9nnAWAtMEkplQ78CuOfKJodKYW7BeOmGWaMua47mzGe5jKjXmlKqR/EGfdCjJvGJhGpxrgp2IEfmds7MG4OYQqj3m8Fss0bRJjuAqE/bAYu7zZ3l1Lq0wSO7f7Z1mPcEPeMGitDGY7i8Nyj5zom0UkqpVYCvwfCPg0nhsZzN1CglMrEELLdv/9Y1GEI5Xhz6f69h7dXJTrfRFBKhZRS7wHvY5iw+vr8bsQwa+5v/t4PNtdHX3P0d7IZw5SpSRAtFIYxSqkWDBv9/SJysogki4hdRI4TkbvM3dIwzDDtIjIVuDKBoWtI/B/lWeA3IpInIrnmfGLFdi8F2kTk5yLiEhGriOwlInO67ygiRcARwAkY6v9MYAZwJ9uikL4BDhaRMWI43H8ZPl4ptRFYBtwqIg4RmYthrthe/gn8UkT2NOeXISJnJHhsDTBKTCe5UioEPAz8RUTyzfGKROQYc//ngQUiMs0Uarf0c66PY0ShnYjhT3Fi3uBNreHoRAYxn6pfxvgMk8WINLswapd/A5PFCIe2iRGQMA1DU9khROQkETnbdBqLiOyH4Q/5PIHPLw1DaDSLSDZ9f36LgREicp3pwE6TXTT8eWehhcIwRyl1D3ADhvOzDuPJ5ycYtmOAn2E8Xbdh/DMlEtp3K/C4GJE0Z/ax7+8xbsArMMwSX5nrus8zyLabfDnGE9+/MJyQ3Tkfw978jlKqOvwC7gP2FpG9lFL/Na9lBYa5pfvN6FwMW3ODOZ/nMLSqfqOUegVDIC0SkVZgFYbNOxHeB1YD1SJSb677OYYT9nNzvHcxnm7DZpN7zeNKzL/9masP+Cvwf8qI1rkWQ9A0YfwOXu/HcD/B0NaqgYXAY1HnacD4Pm/E+IxvBk5QStX3HKbfNAGXAhswAwSAPymlnja3x/38MD47F8bv63PgP72dyPyMjsJ4aKg2z3nYAFzDbouYDhaNZpdGRJ4D1iql+vvkrdFootCagmaXRIycgwkiYhGRYzHCSl8d4mlpNLs8OvpIs6tSiGETzwEqgSuVmQyl0Wi2H20+0mg0Gk0EbT7SaDQaTQQtFDQajUYTYZf2KRx77LHqP//pNSJNo9FoND2Jm+C4S2sK9fUDETKt0Wg0mjC7tFDQaDQazcCihYJGo9FoIgyaUBCR0SLygYh8Zzav+Km5PttseLHB/JtlrhcRuU9ESsRohLLvYM1No9FoNLEZTEdzALhRKfWVGL1Xl4vIfzHqz7+nlLpDRH4B/AKj1slxwCTztT9G9c9+F67y+/1UVlbi8XgG6DI0vZGUlMSoUaOw2+1DPRWNRjMADJpQUEptxSgTjFKqTUTWYNRiPwmj+QcYFR8/xBAKJwFPKCOb7nMRyRSREeY4CVNZWUlaWhrjxo0jqh67ZhBQStHQ0EBlZSXjx48f6uloNAPOhtot3P3WrznANwknXR98BLApH46QG0fIY77cfGWrZo2tNeZ4EwLJHOLL3qE5tUmA15Nq2afgEBacfNsOjRWLnRKSKkav1n2ALzDqvodv9NUYZYDBEBjRzTEqzXVdhIKIXAZcBjBmTM9S9B6PRwuEnYSIkJOTQ11dXz1xNJpdj7LaZm585VTKkzo4tektjul097p/EAsenNwyOod2i5Aa6lotot0ifGtr4vzaku2aj1fglXQbz2bYcbUrMpq/265x+mLQhYKIpLKt9WFr9M1aKaVEpF91NpRSDwEPAcyePTvmsVog7Dz0Z63ZHfluSyu3vXQO5ekdAJQddC1MPqvnjjYn2JPBkYLV6kD5O6h7di4/3fenXDL9ki67/vPbf3L/N/eT/X9rcFgdCc8lpEK8WfYm9319H9Ud1Rwy8mCu/vlnZJ0+OG7XQY0+EhE7hkB4Win1srm6RkRGmNtHYPSMBaOjU3QXqFEMcJennUFDQwMzZ85k5syZFBYWUlRUFFn2+Xz9Hm/t2rXMnTsXp9PJ3XffPQgz1mg00Swtb+R3z1zH6vRKziSHotQiynxNkDOh5ytjFCRnG8JBhPKWcgDGZ/Q0p+a4cgBo9DQmPJcvq7/k7MVn86slvyLLmcUjRz/Cn/f4JXS6cU6cODAX3I1B0xTEeIR8BFijlPpz1KbXMTo83WH+fS1q/U9EZBGGg7mlv/6E4UBOTg7ffPMNALfeeiupqan87Gc/2+7xsrOzue+++3j11VcHZoIajSYu762p4a8v3kfVqKXMCTr45blvcO3HP6espSyh48P7FWf0bGyYk2QIhQZ3A4UphT22d6eqvYpL37mUvOQ8/njgHzm++HgsYqHtvfcASJoypY8Rto/B1BTmY3TYOlxEvjFfP8AQBkeJyAbgSHMZjPZ/ZRgdlx4GrhrEue1U3nvvPfbZZx+mT5/OxRdfjNdrNAgbN24cN998M9OnT2e//fajpKSnrTE/P585c+bo6B6NZpB5aXkltz3zAu0jXyVXCfec/AI2ZxrFGcVUtFQQDAX7HKO0pRSbxcbotJ4tw8OaQoOnIaH5bGzZSFAFueOgO/jhhB9iEeN27Vm3DkR2PU1BKbWE+PU1joixvwKuHsg5/O6N1Xy3JXYUwPYybWQ6t/xwz4T393g8LFiwgPfee4/JkydzwQUX8MADD3DdddcBkJGRwcqVK3niiSe47rrrWLx4h1vgajTfe4558lpqG3KwdcxP+BhfezVTx/+TMovw1CF/ISt7AgATMifgC/moaq9iTHrP4JZoypvLGZc+Dpul5601WlNIhHqPUcYnz5XXZb133XrsY0ZjSUlJaJz+sksXxNsVCAaDjB8/nsmTJwNw4YUXcv/990eEwjnnnBP5e/311w/VNDWa3YZWj58q31KS00dyxJhTe99ZKRwhN6n+Bura7uUtp3D3tMuYUnxkZJewf6CspaxPoVDWUsbU7Kkxt/VXU6h3G0Ih15XbZb133TqSJg+O6Qh2c6HQnyf6oSI6ekdH8mg0O85XG5uwWDuxU8Yfg3/p+X/l74T2WuiohfY6CLh5N9nF9QV5XFp4EMfMuabL7sWZhn+gtLmUQ0cfGve83qCXyvZKflD8g5jbXTYXybbkxDUFd71xjD05si7U2Ylv40bSTzghoTG2h91aKAwHrFYrFRUVlJSUMHHiRJ588kkOOeSQyPbnnnuOX/ziFzz33HPMnTt3CGeq0ewefFG2GSWKVoLUVX9NvuomFOwuSMkzoodS8iAlj8+avyKtZR1XH/W3HuOlO9LJc+X16WyuaKkgpEIxncxhclw5/dIUemgJJSWgFM4pkxMaY3vQQmGQSUpK4rHHHuOMM84gEAgwZ84crrjiisj2pqYm9t57b5xOJ88++2yP46urq5k9ezatra1YLBbuvfdevvvuO9LT03fmZWg0uwyV5UvBNLeXnHo/+UXz+jxm7b/PZXLOHlgt1pjbizOKI+Gm8Qhv71UoJOXQ6E4sJLXB3dBDKHjWrQMGL/IItFAYVG699dbI+6+/jt1T/qabbuLOO++MO0ZhYSGVlZUDPTWNZrfEHwwhjd9GhMKG5g3M60MohFSIDU0bOGXiKXH3Kc4s5vXS11FKxTXzlraUYhEL4zLGxR0nx5XDxtaNfV4HGJrChMwJXdZ5163HkpyMfdSohMbYHnTpbI1Gs9uweksrI6mILK9vWt/nMZvbNuMOuOM6iMF4+u/wd1DTWRN3n7LmMopSi3BanXH3yU7K7pdPIZaT2Tl5MmIZvFu3FgpDSEVFBbm5uX3vqNFoEmJZRSOFti0AZDozKWnuu87QukbDJDM5O76dPmwS6s2vUNZSxoSMCXG3hzo6GNnppNnbTCAU6HVO3qCXVl9rF6GglMKzfj3OQTQdgRYKGo1mN+LbsmrSbcaT+D75+1DaXNpn0tnaxrVYxcrEzNjJYN6yskgEUllzbKEQCAWoaK1gfGbsasG+TZsoP/U0Zv/+dRSKJk9Tr3MKaxPRQiFQU0OopWVQncyghYJGo9lNUErRvulrOi2GzX9WwaxImGhvrG9az7j0cTHNPu5vv6XsB8eTvGYz6Y70uJpCZVslgVAgpqbgXrGCirPPwbdxI46GNqDvXIVYOQreneBkBi0UNBrNbsLGhk7GetbSatrb98nfB4ANTRt6PW5d0zqmZMe+0bpXrwbAt3EjEzInxBUKpS2lQM/Io7b3P2DjBRdiSU4m49RTEZ8fu1/16VcIC4VwwhuAZ53hH3FO1pqCRqPR9MmXFY3MsJTS4kwhxZ7CpKxJCNKrUGjxtlDdUR1XKPhKDSEQqK2lOKM4rvkoVnXUpmefpfInP8E5cSLjFj2La8YMAFI9/dAUkrpqCvaRI7GmpfV67I6ihcIAM9ClswEOPfRQpkyZEhmntra2xz4LFy5ERHj33Xcj61599VVEhBdffHG7r0ej2VVYvrGJfaxltKfmku5Ix2VzMTptNBua4wuFcHTSlKzYQsFbZmgAgdpaxmeMp8nbFNMfUNpcSkFyAamOVFQoRO09f6b6d7eRevDBjH3icWy5uVgzMgBIdfdd/yi8Pdu1rUubd/26QXcyg85TGHAGunR2mKeffprZs2f3us/06dNZtGgRRx5p1G159tlnmWE+nWg0uzvflW9iHFtpTRpPmsMoDTEpa1KvmsLaxrUA8TWFkrBQqGFC5qGAEWU0K2lWl/3KWsoipqPaO++i8fHHyTz7LAp/8xvEZtxmrZmGUMj22RMyH2U5s7BbjOrIIZ8Pb1k5qUf0qCU64GhNYSewI6Wz+8NBBx3E0qVL8fv9tLe3U1JSwsyZMyPbly9fziGHHMKsWbM45phj2LrVaFfx8MMPM2fOHGbMmMFpp51GZ2cnAAsWLODaa69l3rx5FBcXa41DM2xp7PCR1rgKgDabkz0qBd+mTUzMnMimtk14g96Yx61rXEd2UnaPfACAYFsbAbPVrN80H4GhFUQTUiHKW8ojEUqt/32H1MMPp/CWWyICAcBqViEoDKb22Win3l3fxZ/gKy2FYHDQncywu2sKb/0CqlcO7JiF0+G4O/rez2SgSmdfdNFFWK1WTjvtNH7zm9/EzKoUEY488kjefvttWlpaOPHEEykvN2ydfr+fa665htdee428vDyee+45fv3rX/Poo49y6qmncumllwLwm9/8hkceeYRrrjGKgm3dupUlS5awdu1aTjzxRE4//fT+fFoazU5h+cYmZohh728VxQ+fLqWu4u9MuuoIQipEWXMZe+Ts0eO49U3r4yat+UqNm781K4tATS1jUwpx2Vw9yl1Ud1TjDrgpzihGBYMEampx/vDEHv+jYfNRXiCZsr58Cp76LiWzw+Utdob5SGsKg0ys0tkfffRRZHt06ezPPvss5hhPP/00K1eu5OOPP+bjjz/mySefjHu+s88+m0WLFrFo0aLI2ADr1q1j1apVHHXUUcycOZPf//73kfIZq1at4qCDDmL69Ok8/fTTrDYjLgBOPvlkLBYL06ZNo6YmfjanRjOULNvYyExrGaGsYtoCnbg6AgTq6piUNQkgpl/BH/JT0lwS359gOpmTD9ifQF0dogxHcndNIbrbWqC+AYJB7CN6dlazZGQCkOt39m0+6qzvFo66HnE6cYzpvXT3QLB7awr9eKIfKrqXzg4Gg8yaZdgrTzzxRG677TaKiooASEtL40c/+hFLly7lggsuiDnefvvtx8qVK0lOTo4IIjBiuPfcc8+YgmfBggW8+uqrzJgxg4ULF/Lhhx9Gtjmdzi5jaDTDkWUVTVxmK8My6nDa3CuwewIE6xsYkzYGh8VBSVNP02x5Szn+kD9uJrO3tBRxOEjeZ1/a3voPwcZGijOK+bL6yy77hYXEhMwJBNZtAsBWUNBjPEtKMthsZPpsvUYfKaV6lLjwrluHc+LELuaowUJrCoNMdOlsIGbp7PDfuXPnYrVa+eabb/jmm2+47bbbCAQC1Ncb4Wl+v5/Fixez11579XrOO+64gz/+8Y9d1k2ZMoW6urqIUPD7/RGNoK2tjREjRuD3+3n66acH5sI1mp2Exx+kurKCnFADgREzCZk+sUBjIzaLjeLMYtY396yBFC5vEU9T8JWW4hg/Hpv51O+vqWFC5gRqOmto97VH9itvKSfLmUVWUhb+akObto8Y0WM8EcGakUG6R2jyNBFSoZjnbfO34Qv5uuYo7ITyFmF2b01hGLCjpbO9Xi/HHHMMfr+fYDDIkUceGbH/x+O4447rsc7hcPDiiy9y7bXX0tLSQiAQ4LrrrmPPPffk9ttvZ//99ycvL4/999+ftra2Hb9wjWYnsbKqhT2U8dDVlj+FZNOnHGxqQgWDTMyc2OPpHgx/gt1ij1vV1FtWhmv6XtjNp/5AbS3jJxh5COUt5UzPmw4Y5qNwfkKg2gjeiKUpgOFXSHErgipIs7eZ7KTsHvt0z2YO1NcTrK8naZDLW4TRQmEQGYjS2SkpKSxfvrzPcy1YsIAFCxb0WL9w4cLI+5kzZ3bxZ4S58sorufLKK3s9FqC9vb3HPhrNUPNlRSN7W0pRYqUtawyucKBRKESwuZlJWZNYXLaYFm8LGc6MyHHrGtcxMXNiJOwzmpDHg7+ykoyTTsKWnw9AoLaOCfvuBxiCYHredJRSlDaXcsy4YwDwV9cgSUlYMzNjztWakUFSp/F/1OBuiCkUutc98q43M5l3kqagzUcajWaXZnlFE3OdFUj+NFqVf5tQAAINDUzKNJzN0RVTlVK9lrfwlZcbHc4mTsCWkwMiBGpqGJU2CrvFHnEuN3gaaPW1RsJV/dVbsRcUxO25YE1Px9Hpjxwbi+6aws4qbxFGC4UhRJfO1mh2jFBIsayikWmUQdE+tPpaSfZuC4gINjREIpCinc317noaPY19Rh45iosRux1rbg6BulpsFhtj08dGyl1077YWqK7BFsOfEMaamYG1zQ3Ez2ruLhS869Zhy8vDlt1TqxgMtFDQaDS7LCV17WR4q0gJtkLRLEMoRFWTCTQ0UpBcQJo9rUtY6rom08kcT1MoKwWLBce4cQDY8/LxmyHZxRnFEU0hHHkUTlzzV1dHfBCxsGRkIG3bzEexqHfXY7fYSXcYyW6enVTeIjLHnXYmjUajGWCWVTQxU8y8gZH70uZr62I+CjbUIyJMzJrYpdxFpLFOVrxw1DIco0djcTgAsOXnE6g1spuLM4upbK/EG/RS1lJGij2FguQCI3GttjYSrRQLa0YGqr0Dp7L2aj7KdeUiIqhAAN+GkkHvoRCNFgoajWaXZVlFI/s5K1C2JMjfg1ZvayT6CAxNAWBS5iQ2NG+I5Nqsa1rHiJQRXRzP0XhLS3BM2NYbwVZQQMAsRFmcUUxIhahoqaCs2ah5JCIE6uuNxLXC3oRCJgBFZPaqKYRNR76KCpTfv1PKW4TR0UcajWbY88WmMpKkp039i/JGrnRWILl7g9VOm6+NFJ/h5LXm5RJoNG68k7Im8fz656nprKEwpZB1jevi+hNUIIBv4ybSDjs8ss6Wn0ewsZGQzxfxH5S3lFPWUsbckXMBCFRXG/v2KhQMITQymNGrpjAydSSwc8tbhNFCYRCoqanh+uuv5/PPPycrKwuHw8HNN9/MKaecMqDnWbt2LRdddBFfffUVf/jDHwakGqtGM9z48at3sLTlaTpKbyDky++yzUqQ8cklUHQRAK2+VrIDDiwpFmy5eQTrjRtvuNVmSXMJmc5MKlorOHLskTHP59u0Gfx+HBO2NcwJh6UG6+oYVzgOi1j4tu5b6tx12yKPthpCoXdNIVwUL4XVvWgKe+ftDRjlLbDZcI6P3eZzMNBCYYBRSnHyySdz4YUX8swzzwCwceNGXn/99QE/V3Z2Nvfddx+vvvrqgI+t0QwH/rzkZZa2GFn21x+XzZ5ZXcvHp7esx/aWB4r2BaDN18ZYvw1LWgq2nBwCjab5KFwDqWkDWc4sQioUP5PZ7KHgjDIfhZ3H/tpakouKGJU6inc3Gb1LJmQa+wVqEhEK4aJ4Lho9G3tsD4QCNHmaukQeOYuLEdO3sTPQPoUB5v3338fhcHTJWh47dmyk6mgwGOSmm25izpw57L333jz44IMAfPjhhxx66KGcfvrpTJ06lXPPPbfPWkP5+fnMmTMHu71n8o1Gs6vz3w3f8OiGP2ILGdVCR+WGOHxqQZfXbLtZsXTkNqGQ5rdiSU3BlpNN0CwRk+HMIN+VT0lzSZ+RR16zh4JjfE9NIVCzza9Q3VEdeQ+GpiBJSVgyYvspYJtQyPY7afQ09vgfb/I0oVCRjms7s7xFmN1aU7hz6Z2RJhoDxdTsqfx8v5/H3b569Wr23XffuNsfeeQRMjIy+PLLL/F6vcyfP5+jjz4aMLKeV69ezciRI5k/fz6ffPIJBx544IDOX6PZFahorOVnH12HiJN/HfMgC949NbYNvmo5JGVAttnLwAxJtaakYs02NAWlFCISabiTak+NdGWLhbesFNuIEVhTUyLrbFGlLgDGZ47nw8oPcVgcFKUaBSv9NdXYCwvjJq4BEYGR6bXiD/lp9bV2cXZH5ygEW1oIbN2608pbROa4U8/2PeTqq69mxowZzJkzB4B33nmHJ554gpkzZ7L//vvT0NDAhg1GqNx+++3HqFGjsFgszJw5k4qKiiGcuUYzNHT6vZz92lUELS3cMucuZhVNItmWHDtap+orGLkPWIxbWZuvjWQPWFJTseXmoDwelFkgb1LWJEqbS/mu4TsmZ03GIrFvf77SMpzFxV3WWTMzwW4nUGvkKkzIMExG4zLGYbVYAQhsre7VyQzbGu2kewzB0V3QRYRCci6+TUbFVUe3uQw2g6YpiMijwAlArVJqL3PdrcClQJ2526+UUv82t/0S+DEQBK5VSr29o3Po7Yl+sNhzzz156aWXIsv3338/9fX1kVaaSin+9re/ccwxx3Q57sMPP+xSptpqtRIIBHbOpDWaYcQ5L/6KDss6Tim6kdP3PACWPUaOz0PDykWw7NWuO9ethfnXRRZbfa04vSEsaWlYs40qo4GGBhwpKUzMnIgv5OPbum85Y/IZMc+tQiG85eVknn5al/Uigj0vD39UWGr0XzCqqKbsv3+v1yZWK5b0dJLdhtmowd3QZYxoTSFQZ5S3sOXl9xxoEBlMTWEhcGyM9X9RSs00X2GBMA04G9jTPOYfImIdxLkNGocffjgej4cHHnggsi7c3hLgmGOO4YEHHsDvN+qfrF+/no6Ojp0+T41mOHLz2w9R5nuHqa4TuL14PDx4MCy+jhwsNDqSIGdC19e0k2HmjwDjgavV14rDEzB8CrnbhAJsczYrVFx/QmDrVlRnJ87iCT22GbkK2xLYHBZHZJxI4lph/GzmMNb0dJLi1D8KC4WcpBwC9ca5bHk7txTOoGkKSqmPRGRcgrufBCxSSnmBchEpAfYDYrciG8aICK+++irXX389d911F3l5eaSkpEQqoV5yySVUVFSw7777opQiLy+vz+ih3/72t8yePZsTTzyxy/rq6mpmz55Na2srFouFe++9l++++450U0XVaHYlXlj1Cf/e+g/yQhN5NlAOT5wImWPgjMfJrvmATW2b4KSn4h7vDrgJhALY3WL6FIy8hqApFIozirGIhZAKxc9kLjPKVzgnxhAK+fmRiqUp9hSe/+HzEX/CtsS1+HWPwlgzMpB2I8Ouu0ms3l1Pmj2NJFsSbaaTfGfVPAozFI7mn4jIBcAy4EalVBNQBHwetU+luW6XZMSIESxatCjmNovFwh//+MceTXAOPfRQDj300Mjy3//+98j72267LeZYhYWFkZaaGs2uzqPLXsJKiFerlmCzOuCIW+CAq8CeRE7rCr6ujV1+Pkybrw0JKayeAJa0NKO6KduympNsSYxJG8PG1o29lLcwI48mxBYKHUuWRJbDoahgaBhAYppCRga0d2ARS0yhEG6uE6yvx5qZuVPDUWHnO5ofACYAM4GtwD39HUBELhORZSKyrK6uru8DNBrNsKe2zUNm6zKygwEy9j4LrvkKDroB7EkA5LhyaPY2EwjF97O1+lpxmcXwLKkpkSfsYOO2G+/03OlMzppMsj055hi+0jKsWVnYsrJ6bLMX5BPq6CDY3tPc21vHte5YMzMItbSQ5cyi0dPYZVt0iYtAXf1ONx3BTtYUlFKRzu8i8jCw2FysAqLjw0aZ62KN8RDwEMDs2bN102CNZjfg9a82k2Wtpd2WCif9vcf2nKQcFIpmb3OX3sXRRBfDs6amIg4HlvR0AvXbhMKv9v8VvpAv5vFgaArRmczRbGu2U4s1tWuGsd/suNZbhdQwlowMgi0t5LhG9NAUGjwN7JG9h3Ge+nqsQ1BaP2FNQURii9Z+ICLRYvQUYJX5/nXgbBFxish4YBKwdEfPp9Fohj9KKdYtfQefNUhmSuxIm3CHsnhF5MDMUTCFgiU1DcDMat52TKojNWa3s/A8fKWlMZ3MALb8rrkK0QSqaxCXq9fEtTDWsFBwZMd0NEe34bTl5vU53kDTp1AQkXki8h2w1lyeISL/SOC4ZzEcxVNEpFJEfgzcJSIrRWQFcBhwPYBSajXwPPAd8B/gaqVUcHsvSqPR7DqsqmplZst7tFitZGSMiblP2M7em1Bo87VFeilYUlMBsOZkR+of9UWwsZFgS0tMJzNEawo1PbaF+yj0lrgWxpqRCaEQhZLR5Xo6/Z10+DvIdeWilDKFwvA0H/0FOAbjaR6l1LcicnBfBymlzomx+pFe9v8D8IcE5qPRaHYjXlpWzk+tS3nCWcCkOE/xOUmmUIhTWRRMn4LZdS2cjWzLzok4j/siUt4irqawzXzUnUB1da99FKIJJ7AVBFJo8DREMq7D15bryiXU0Ylyu4dEKCRkPlJKbe62Sj/FazSaHcYbCFLzzTtkSRutFiHdGTucOqwpdHfMRtPFfJRmmo9ycyIhqX2xrRBebJ+CNTUFS0pKJIEtGkNTSFAoZBomptxgEt6gl86AkccU1hpyXbkEhyhHARITCptFZB6gRMQuIj8D1gzyvHZpampq+NGPfkRxcTGzZs1i7ty5vPLKKwN+noULFyIivPvuu5F1r776KiLCiy++OODn02gGmvfW1HJ44GO89jTag95IC8rupNpTsVvsvfsUvK1kBIzwzYj5KDuHYHMzykwW7Q1vaRmW5OReS1XY8vMjRfHCqECAQF1d4ppCuCiez5hr+Jrq3IYgyHXlGnkPMGw1hSuAqzHyBqowwkmvHsQ57dKES2cffPDBlJWVsXz5chYtWjRo+QTTp0/vkhPx7LPPMmPGjEE5l0Yz0Ly6rIzjbMvo2OM4gLid0ESEHFdOr+ajNl8bWUGjVIwlxRAKkazmpqY+5+IrK8UxYUKvfgGjLWdXoRBJXEtUU4gqigfbTGKRbGZXTkQoDLvoI7PUxF+VUucqpQqUUvlKqfOUUonpY99DdmbpbICDDjqIpUuX4vf7aW9vp6SkhJkzZ0a2L1++nEMOOYRZs2ZxzDHHsNVMsnn44YeZM2cOM2bM4LTTTouU4liwYAHXXnst8+bNo7i4WGscmkGjttWDlLxHKp20TzK6nMXTFMDwK/TlU8jw20AES4oRLNk9q7k3vCWlPQrhdcdWkE+gpqujOdxxzZ6gphCOUEr1GMthTaHeXY9FLGQ5swjUmZpC3s6PPurV0ayUCorIWBFxKKXiB/cOU6r/+Ee8awa2dLZzj6kU/upXcbfv7NLZIsKRRx7J22+/TUtLCyeeeCLl5UaNeb/fzzXXXMNrr71GXl4ezz33HL/+9a959NFHOfXUU7n00ksB+M1vfsMjjzwSEVxbt25lyZIlrF27lhNPPJHTTz+9X5+RRpMIr3xdxfGWTwkmZdOSPxXoQyi4cqjrjJ+wavRSsGFJTY087XfPao5HsK2NQG1tzEzmaOz5+bTW1UWcw2D4E6D3NpzRhDWF5M4QpG8TCg3uBrKTsrFarIamYLNF9t2ZJBJ9VAZ8IiKvA5FUPqXUnwdtVrsRV199NUuWLMHhcPDll1/yzjvvsGLFisgTeEtLCxs2bMDhcERKZwOR0tmJ9FM4++yzue+++2hpaeGee+6JlNBYt24dq1at4qijjgIMLWWEmXG5atUqfvOb39Dc3Ex7e3uXqq0nn3wyFouFadOmUVPTM/xOo9lRlFIsXlbCC9avsO51Lq2mszWeoxmMXIW1DfEf8oz+zJaIPwG2CYXorOZY+MI1j+I4mSPj5eeD30+wuTmS9RwWCr11XIvG4nQiSUk4OnxIunQxH23LUajDlpODWHZ+d4NEhEKp+bIAaYM7nYGltyf6wWIoSmfvt99+rFy5kuTkZCZP3lbTRSnFnnvuyWef9awruGDBAl599VVmzJjBwoUL+fDDDyPboueRiAlLo+kvKypbGNvwMUkOL+x1Gq2+VgAyHPGfjHOScmj0NBJSoZi9EFp9raR4pUtzHGtYU+gjV8FbGhYKvWsKkQS2mpqIUAhsrTYS1/pRiNKakYFqbSNzXGYX81E4ymqochQgAUezUup3SqnfYdQpuidqWRODoSqdfccdd/QosjdlyhTq6uoiQsHv97N69WoA2traGDFiBH6/n6effnqHz6/R9IcXl1dyku1zQqmFMGYuLd4WoHdNIceVQ0AFaPW2xtxulLlQXTQFi1nuoi9NwVtagtjt2E1NPR6xchX8NTV9dlzrjjUjg2BrC9lJ2V00hTyX4UMI1g1joSAie4nI18BqYLWILBeRPQd/arsm4dLZ//vf/xg/fjz77bcfF154YZfS2dOmTWPfffdlr7324vLLL+9TI/jtb3/L66+/3us+xx13HIcddliXdQ6HgxdffJGf//znzJgxg5kzZ/Lpp58CcPvtt7P//vszf/58pk6dugNXrNH0D48/yHvfbOAw6zdY9joVLNaIptCXoxli5yoEQ0Ha/e1Gg50ooSAiWHNy+tQUfKVlOMaPR2y9G0/sBT2FQmDr1oSqo0Zjzcgg1NxiRFS5GwipEA2ehi4lLqxDkKMAiZmPHgJuUEp9ACAihwIPA/MGb1q7NjurdPaCBQtYsGBBj/ULFy6MvJ85cyYfffRRj32uvPJKrrzyyl6PBWhvb495bs3uS2OHj9Agmg3fX1vLXN/n2Bx+2MvocNbqbSXJmoTDGr9MdKTUhaeBYrra/tv9xu/U4fZjTUvtss2Wnd2l/lEsvKWlJO3V97Ou1YwG8kf52vw1NaTMndvnsV3GyczAV7GRnKSprGpYRau3lUAoYJS4CIUINDQMmaaQiFBICQsEAKXUhyKS0tsBGo1m1+TCl+7iy/p36Cy/flDP86zrC1TGGKRoFmD4A3ozHUHvRfHCJiWb2x/JUQhjzc3ptf5RyOPBX1lJxkkn9Tlvi8OBNSsr0oFNBQIJd1zrMk6kUqqhKYQT13JcRrIdwSC2nOErFMpE5P+AJ83l8zAikjQazW5ERX0HS7eswJZew60nTsEqg1NZ3+lr4oAPVyB7XgOmHb7F29Kr6Qi6agrdafUbQsHS6e1iPgKz/tG69XHH9ZWXg1J9Rh5FxisoiOQqBOrrIRRKqONaNNYoodAZ6KSyzUhuzU3KjcpRGL5C4WLgd8DLgAI+NtdpNJrdiLvfWYfNbjh8jxuxmYI4xel2mPX/BRWMmI7A1BT6EAqZzkysYo2rKViDCovXh6W7+cisfxSdWxBNOPKorxyFyHj5eRGfgt9MBrX316eQnoHyeskV45rXNa0DzLpHZVvMeQ9ToWC2y7x2J8xlwIj35WsGHh2yunuwsrKFxSu2MmnKFqqBhmfPoMDXd72g7SZ3MhROjyy2+loZmTqy10MsYiErqWe3MjAij5LM9FprN03Bmp2D8vsJtbVFKpRG4y0tAYsFx7hxCU3dlp+PZ41R/i2sMdi2Q1MAyA0Y4d/rmwxNxqh7tMIYc7gKBRH5L3CGUqrZXM4CFimljun1wCEiKSmJhoYGcnJytGAYZJRSNDQ0kJSUNNRT0ewgd729lr1cjdTiASw0HP4ryJwyeCcs2CtiOgJDKOzh2KPPw7KTsmNrCtEVUrv5FGw5hsYTaGiIKRR8JaU4xozBkmAvZHt+AcH6BpTfH5W41k9NwayUmmUWxVvftJ4kaxIp9hQa68J1j3Z+iQtIzHyUGxYIYGgOIhK7PdIwYNSoUVRWVqL7N+8ckpKSIlnYml2TJRvq+XhDPS8Xv8uFZgZtQ24xTDy+z2OVUvy7/N8cNvqwuH2PE6HF29Knoxni1z9q87VFlc3upilEspobYfz47ofiLSvDEaexTixs+fmgFIGGBiNxLTm5X4lrsE1TSPcan/em1k0UpRYhIgTq641kuJQdbna5XSQiFEIiMkYptQlARMZi+BaGJXa7nfExvniNRtOTUEhx53/Wsk9GB5n1/4Ei43mvtxLV0ZS3lvOLj3/Bj/f6MdfNum675uAP+XEH3H36FMBwNm9q29RjfauvlRSfBQj2MB/ZeslqVn4/vo0bSTviiITnawvnKtTUGIlrCXZciyYsFFLdxq1Uobq14cwdMktHIoU1fg0sEZEnReQp4CPgl4M7LY1GszP496qtrKxq4U8jPqDBuu120Fs10mhqOgyb+gvrX6DT39nH3rEJh5MmJBSSjBDO7r6sNl8bOSEXQM/oo17qH/k2bYJAIG4LzliEs5r9tbUEtm5NuDpqNGGhIG0dpDmM6kFd6h4NkT8BEitz8R9gX+A54FlgllLq7cGemEajGVz8wRB3v72OuXl+Jmx+idpio/iiRSwJawrh+PpWXyuLyxZv1zwidY/i9FKIJtuVjSfoiXQri4zhbSUraPi2LKldS7RZwzWKYmgKfbXgjIU9qtSFv6YGW4J9FKIJl88ONrdEMrXDIbfBIax7BL0IBbNkdgaAUqoeo0Lq0cAFIpKYR0aj0QxbnvtyMxUNndwx8n9IyE/92AMAKM4oTlhTCJeynpg5kSe/e5KQCvV7HpG6RwlqCtDTvNXqbyXLH+661jW3Vmw2rJmZMbOavaUlADiLEzc5W3NywGolsHUrgdra7dIULCkpYLVGchUgSlOoqx+yHAXoXVN4HkgBEJGZwAvAJmAG8I9Bn5lGoxk0On0B/vreBo4YI4wpWwTTz6BWQrhsLsakjUlYU6h315NsS+aS6ZdQ0VrBkqol/Z5LpO5RIo7mOAlsbb420gOGi7S7TwHMrOYYPRV8pWXYR47Ekpy4U1csFmx5ebhXr4ZQaLs0BRGJFMULC7pcVy7K5yPY3DwkHdfC9OZodimltpjvzwMeVUrdIyIW4JtBn5lGo+lCaUMt937yBuOT+u6x0Rfratqoa/Py+z2WILVuOOhG6tY8Qq4rlxxXDt/UfZPQOHXuOvKS8zh67NH8edmfeeq7pzh41MH9mksixfDCRIriubve4Fu9raT5bWC1Ii5Xj+Ns2TkEYnRf85aW9ivyKDJefj6elauAxDuudSec1ZydZAiAXFcugUbjuobSfNSbUIh2fR+O6VxWSoV0/L9Gs/P51TsL+c73JG+W2lD+Hb9pXLRvJiPWPgnTToK8KdQtryPPlUeOK4cmTxOBUACbpfcAxbrOOnJduditds7Z4xz++tVf2dC0gUlZkxKeR9jRnJBPIVz/KIamkOJL79J1LRpbbg6e79Z0WaeCQXzl5aQccEDCcw1jL8jHs8JMMtsOTQHMSqktLeS4jHyQXFcuga1miYshylGA3oXC+yLyPLAVyALeBxCREcAu15pTo9mVqW3zsKK6Els2/OuS0Rw+5vAdH/TDO+C7Njj4JsB46p+aPZWcpBwUimZvc8TOHY96dz3TcqYBcMbkM3jw2wd5as1T/G5e4i1XWnyGTyEchdMb2a6eRfGUUkbymi8da0rsWp3W7JzIU3gYf1UVyuvtV+RRGFvetlStHdEUAnV1jE0fi81iY0TKiG3ZzMPUp3AdRr2jCuBApVQ4570QI0xVo9HsJB7/tAJlaQOgtLl0xwf0tMLn/4Apx0PhXoDx1B/WFCCxXIU6d11EcGQ4MzhxwoksLl2csE8CDE0h2ZaM3WLvc1+7xU6GM6OLpuANevGH/CR5FJa02ILFlpNNqLWVkG/b86y31Iw8SrDmUZfxzAgkSU6Oe86+sGSkE2xp4eixR/PGyW8YTYTqw5rCMDQfKSMQuEdTAKXU14M6I41G04V2b4AnP9vIiPFu6oHSkregYweV9S1fg6cFDv4ZAB3+DjoDneQl58WN8OlOh78Dd8BNXvI2U8e5087l+fXP88L6F7hixhUJTSWRstnRhNtyRh8P4PQEsaTGNkFFZzVbzF7KPlMo9NWCMxa2AqOsRX87rnWZU0YmwZYWrBYro9KMqgBBUyiE5zsUDE5tXI1GM2A89+VmWj0BJssm6hWU1q+GFe/t+MBTT4CifYFtoaVdNIU+wlKjjwlTnFHMgUUHsmjtIi7e6+Jem+aEafW19tqbuTvhHgRh2nyGBmX3+LFkxzYfRWc1202h4C0pxZaXF7MeUl/Y8o1r7m/No2isGRmE2ttRgUCk41ugrh5rRkbCdZgGAy0UNJphjD8Y4tEl5Rw21k6VvxlsVsqTUwneXI7VYt2xwaNs+OEktP5oCuFjuvsdzp92Ppf/93LeKn+Lkyb23bim1ds/TSE7KZu1jWu3HW9qCrZOP9bUeOajnlnN3rKy7TIdAdhNTaG/1VGjCWc1B9vasEUS7IauDWeYRMpcRBCRLBHZe7Amo9FouvLmiq1UNbv5Vd4nNFqETHsa3qCPLYEOSMrYsZdl279/+Kk/35VPij0Fp9XZp6ZQ7zZMHdGaAsDcEXMjyWyJlFZPpJdCNOFSF2HCmoKl09OjxEWYsDkmYOYqKKXwlZZul+kItvkUwlrH9hCulBpsbo6sM+oeDV3kESQgFETkQxFJF5Fs4CvgYRH58+BPTaP5fqOU4sGPytgjz0nhxmfxWizMGWmET5Y0lwzouaI1BRHpceONeUzntmOiERHO2+M81jWtY1nNsj7P3ertp1Bw5dDub8cbNMqihjOipdPdI5s5jC3biFoKNhiCLFBTQ6ijA0eC3da6Y01PZ+Q9d5N51lnbdTxs0xRCLS2RdYEhLnEBiWkKGUqpVuBU4Aml1P7AkYM7LY1Gs6SknjVbW/nd+O9oNG/QswtmA1DaMgARSFHUddaRZE0i1W48aee4ciKaQNxj3HU4LI6YN/Tji48nxZ7C2xV9l0lr9bVGchRUqO8yGd0T2Np8bViDCrw+rHEigSwpKYjLFdEUwjWPnBMm9nm+eGQcfzz2gu3vIhAxH3UXCkPoZIbEhILNzE04E9i+ilcajabfPPi/MvJTHcze8hSNecbNa1z6OPKT8wcmLDWKWndtREuA+H0LoglnM8eKvkmyJTEqdRTVHdW9juENevEEPaQ70ml6/nlKjjiSUEdHr8d0T2DrrcFONLbsbAKmpuArCwuF7dMUBoJwD4awUAh1dKA6O4c0RwESEwq3AW8DJUqpL0WkGNjQ10Ei8qiI1IrIqqh12SLyXxHZYP7NMteLiNwnIiUiskJE9t3eC9JodgdWVbWwpKSeW/bYiqV+HQ1TjwOM5K2JmRMHXCjUu+u7+Aa6R/jEPKazvtfktsKUQmo6a3odI7pstmfVagJbt9L8yqu9HtM9j6LN1xZVITW+UIiuf+QtKcWakTGkoZ/WzEzAqJQKRHIUhrLuESRWOvsFpdTeSqmrzOUypdRpfR0HLASO7bbuF8B7SqlJwHvmMsBxwCTzdRnwQGLT12h2Tx7+uIwUh5VjWl+AtBE05k0GjCf44oxiylvKt6siaTzqOuu6+Aayk7Jp8jYRDAXjH+Ou6+FkjqYguaBPTSG6GF6gthaAxieeQAXjnzcsFMK5Cq2+VnJDRkG7eD4FMOsfmVnNRs2jiUPasjds6gprCtsS14a/o/ku09FsF5H3RKRORM7r6zil1EdA97KEJwGPm+8fB06OWv+EMvgcyDRNVhrN947Kpk4Wr9jKdXu5sW38CPa/ggazFERmUiYTMyfiCXqoaq8asHPWdtb20BRCKkSztznuMdHZzLEoTCmk2duMJ+CJu0+kl4Ijg0BtLZKcjH/TJto//DDuMd3NR22+NnKCRhG8eD4FAGtONsH6eiPyqKQEZ/HQmY7AKOltSUsj2Gp8BgGzN/NQm48SyVM4Wil1s4icglHy4lSM7mtPbcf5CpRSW8331UA486MI2By1X6W5bivdEJHLMLQJxowZsx1T0Gh2HqtrNnPe4kvw1h8JHdMTOiYQVAhwbugNcKTCrAU0fPt3Mp2Z2C12JmQaYZRlzWWMThu9w3OMzmYOE53AFn4fjSfgoc3X1iPyKJqCFOPfu7azljHpsf9XI70UnOn462pJP/poOpZ+QePCx+O2yHTZXCTbkiPmo1ZfK+MDTqAPn0JOLoGmJoINDQRbWrar5tFAY1RKbQYYFiUuIDGhEN7neOAFpVTLQKhcSiklIv3u9ayUegh4CGD27NnDtle0RuPx+7jozZ8SsG1hzJi1HJR+fMLHzs7qJPnd12C/y8CVSaOnMfKEXJxpPOGWNJdwyOhDdniesTKTuySwZfU8Jl6OQjQFyYZQqO6ojisUwppCmiWFzoZG7CNHkH3e+dTedRfu1atx7blnzOOifR5tvjYyArEb7ERjy8mGQIDOr74C+tdtbbAIl88Gow0nVmvE1zBUJCIUFovIWsANXCkieUB8fbB3akRkhFJqq2keqjXXVwHRjzyjzHUazS7Lgldvx23dQI6jiFa1hp8fN7nPUtQR3vkNKAX7G/WDGtzbntjTHenku/IpaykbkHlG5yiE6avURVgo9GU+Anp1NocdzantATpDIWz5+aQffzz1f/87jY8/TtFdd8U8Ljo6qs3XRro/E+jLfGRcU+fSLwGGiaaQTsh0NAcbGrBlZyPWHcxU30EScTT/ApgHzDYrpXZi+AC2h9eBC833FwKvRa2/wIxCOgBoiTIzaTS7HH/77DVWd77KGPvh/HLu9bT521hVv6rvA8GoYLr8cdjzZMgaC9BFUwBDWxioBLY+NYVYx8QQJN3JTzZi+HtzNoc1haRmo+eyLT8fa1oaGaedRuu/38JfUxvzuBzXtqJ4RoMd40baW/SRLSIUlmJJTsa2A9nIA4UlWlOoG/oSF5CApiAiycBVwBgMW/5IYAp95CyIyLPAoUCuiFQCtwB3AM+LyI+BjRi5DwD/Bn4AlGAInYu241o0mmHB8qpSHlrzB+xqFM+cfht8+DsswKdv38BMawLJTh214G2FuT+JrGpwN0Ru1GD0RH5pw0uEVAiL9KtaTQ9i3eDTHenYLfa4mkJYkPSmKbhsLjKdmb1qCi3eFtLsaYTqjfOE+xRkX3A+TU89RdMzz5B//XU9jstOyuarmq8IqRDt/naSfQI2G+J0xj2X1cxq9q5fT9L06UMaeRSmq/lo6LOZITHz0WPAcgxtAQyzzgv0IRSUUufE2dTDe2SW6b46gbloNMOaDq+Xy9++HkTx9yP+QsaXD8CXj7DXmPF8Emrkqg5/34MAzLkkUsHUF/TR5m/r4vAtzizGHXCzpX1LpOzy9hLOZk6zbzO9iEivuQr17nqsYu2ivcSir7DUcNnscDhquKaQY/Ro0o48guZFi8i94nIs3Vps5rhyaPY20+xtRqFI9iqscbquhYm+4W5vzaOBxpqRSbC1FaUUgfp6nJMnD/WUEhIKE5RSZ4nIOQBKqU4ZDiJWoxmGnPfK/+G1lnP++P9jnsMHH/0Jpp/J3OIZPLzyYVouWpxQ28lowmaS6BvwxEwjw7mspWzHhYIZWtr937q3rOY6dx05STl9ail9JbCFi+EFqmrBYjGcwSbZF15I23/fpeW118g6++wec1MoNrVuAsDpDfVqOgKzrITFAqHQdtc8GmisGRkQDBJqayPQ0DAsNIVE9E6fiLgABSAiEwDvoM5Ko9kFueuj5ynxvsVE53HcPP8UePUqcGXDcXcyv2g+IRViafXSfo8bvjFHm4+KM4yb2kBkNte56yL2/87ly6l/+GHjfK6cSH2hWMfkJvd9AytILqCmo3fzUbozHX9tLbacnEhfAQDXrFkk7bUXjY8/0aMmUlhrKm8pB8INdnoXCmK1RkxIO1LzaCAJ1z/ybdoMfv8uIxRuAf4DjBaRpzEykW8e1FlpNLsYn2xcw5Olf8IZHMdTp9wOH98DNSvhh/dCcjZ75e5Fqj2VT6o+6ffYYRNOuD8xGK0v81x5A+JsruvcloTW/Pzz1N37V1Qg0Gul1LrOOvJdfftHClIKaPI2xU1gi2gKtbUR01EYESH7wgvxlZfT/tFHXbaFBWRFawUANncAax9CAbZVSx3KmkfRhMtn+0qN73GoE9cgseij/2IkrC0AnsWIQvpwcKel0ew6VLc1cfW714Cy8OAx95LStN40G50BU43cBLvFzn6F+/HZls8S6jEQTdh8FK0pgOFXKGve8bDUaE3Bt7kSgkH81dWRCJ9Y5TTq3fUJaQrhsNTazthRROGy2YHauh5CASD92GOwFRTQ+PjjXdaHTWkVLRUA2Dq9fWoKYGQ1i8OBfdSOmdwGinDXt3DV1l1FUwBIApqAVmCaiBw8eFPSaHYdfIEAZ7x0DQFrHTfO+AOzCsfAq1eaZqOuMfbzi+azpWMLG1s39uscEU2hm1N3YuZESltKd6gGUqe/kw5/R0RT8FdWRv7mJOUQUIFILkEYf8hPo6ex18S1MNEJbN1RSnVxNMcSCmK3k3XuuXR+9jm+iorI+oj5qNUwH/XWYCca18yZpBx44JDnAoSxmOYjr9kveqiL4UFiIal3AmcBq4Hwr09hlLrQaL7XLHj1NprlW47Ov4IFs46ED++E6pVw9jOQ3PUmPnfkXAA+2fIJ4zLGJXyOBk+DUdrBntxlfXGGEYFU3VHNyNSR2zX/cDhqfnI+IY8nEgXkr6wkZ9a2BLbMpMxt8zGFVG/hqGF6S2BzB9z4Q34yJZVgY2Ok73F3Ug8+iLo//xnPmjU4xo0z1tlTcVgcbG4zq+N0urGk9S0U8n/60z732ZlYMzIB8EbMR0NbDA8S0xROBqYopY5XSv3QfJ04yPPSaIY9f/jwGVZ2vMIY+2HcfcyVhjD46K4uZqNoRqeNZnTaaD7b8lm/ztM9cS1MOAJpR/wKYbNOrisX/5YtkfU+U1OAnglsiZS4CNNbAls4cS3bbUQ9xdIUABxjx4II3rJtprJwyGwgFMAqVlRbe0I+heGGNcMwH/k3VyJJSVhS4pfp2FkkIhTKAPtgT0Sj2ZV4Y82XPFt+N67gBJ4/7U9Ygt64ZqNo5o2cx9LqpfiDCeYr0LXERTTRhfG2l/ANPj85H//mbTUp/ZVVcUtdxGvDGQuXzUWGMyOmphCpkNpqGCDscYSCxeXCPnIkvrLyLuvDgjLLkory+3sthjdcsSQlIUlJEAphy+0ZFjwkc0pgn07gGxF50GyEc5+I3DfYE9Nohisb6rfy609vxKJSePKH/yBl8xL4x1xDUzjhLz3MRtHMGzkPd8DNN3XfJHy+Rk9jDyczGBFIua7cHWrNGa0p+Ex/gmPChIhPAXpqCmGTUyLmI4DC5MKYYalhX0VaiyEg42kKAI7iYrzlXYVfWGjlhoyn60TMR8ORcFjqcHAyQ2JC4XXgduBTjMzm5UDf3bg1mt2QDq+Xc1+/mpCljf8389dM+d//wVOngljggtdgjxN6PX6/wv2wiY1Pt3ya8Dkb3A1xM4cnZEzYoVyFenc9TquTdEd6xIThmjEDX1Ul6c50bGLroSnUu+sRJKb2EouClAKqO3uaj1rMHhHJLUa4am9CwVlcjK+8oku+QlhohRvs7IrmI9gWgWTNHdrezGESEQqZSqnHo1/ELKar0ez+nP/Kb3FbN3B1yjyOf/syWPM6HPILuPJTKD60z+NTHansnbd3wkIhpEI0eZvi3oCLM4spbS7td5hrmNrO2kg2s7+qEvuoIhyjRxGsqwevj+yk7JiaQlZSFnZLYlblvjQFZ1MH2GxYs+LfVhzFxSi3m0D1NuES/kyyE2jFOZzZFTWFC2OsWzDA89Bohj1/++w1Nnj/zQ877Vz53VMwYm9DGBz2S7AnJTzOvJHzWNOwJpJ/0BvN3mZCKhRXU5iYOZHOQGefbS/jUe+u75Kj4CgaFYnh91cZfoUemkIfvZm7Ey+BLexTsDW2YcvLQyzxb0fO4vEAeEu3mZAiPoVwg53U+GWzhzOWzLBQGPrII+hFKIjIOSLyBjBeRF6Pen1AzzabGs1uzZraSh5a8/8Y6xN+21wHJ/8TLnwDcif1e6x5I+ehUHy+5fM+9w0/pcfVFMLlLrbTrxBuw6mUwr95M/bRo7EXmUKhspJsV2xNIZHIozDxEthafa0IAvVNccNRwzjM1pm+KL9C2HyUHjA0lt4a7AxndiVN4VPgHmCt+Tf8uhE4ZvCnptEMDwLBIBe/eQM2cXNfbRVJJ/0dZp4D2xkpMi1nGhnOjIRMSPGymcOEw1K3169Q564jLzmPYHMzoY4O7KOKsI8qAraFpfaIPuqjN3N34iWwheseBetq40YehbFmZ2PJyOgSlhppOuQ30q12WZ+CmaswHEpcQC/Ja0qpjRg9D+buvOloNMOPKxf/mXbLGm6pb6J42hkwbXt7TBlYLVYOGHFApORFb2GIEU0hjlDITMokOyl7u4RCOJs5z5UXyWR2jB5tmHKcTiMsdapR/yg8z5AK0eBuiJiclFK0f/AhKfvvFzfGPiwUuoelhuse+WvrSJ6zX69zFRGc48d3CUsNfyapfuPZ1tJL17XhzC6jKYjIEvNvm4i0Rr3aRKQ13nEaze7Eiys/4fOmpzi4U3GqJQOOu3NAxp03ch617to+E89ilc3uTrjcRX+Jbq4TFgr2UaMQEexFRZGwVH/IH7H/N3oaCapgRFPwrt9A5VVXUXXTzT0qmYYpSIkvFLIllVBLS6+RR2EcE7qGpY5IHUGaPY28oBmSuotqCrb8fBDBVjhiqKcC9G4+OhdAKZWmlEqPeqUppdJ30vw0miFjS2sjty/9NZlBK3+sq8Jy8j8hqX+9EOIxb6TRs6ovE1KDpwGb2Eh3xv+XK84wCuP1NwIpug2nb7MpFEx/gn1UEb6qyh4JbJFsZjNxzbt2DQDt779Pw0MPxTxPOIGtu/mozdvGCLfhJE5EKDiLiwnW1RNsNQRUij2FD8/6kHG2AsRux+JwJHjlw4v043/AuGefwV6QQFe+nUBvQuGV8BsReWknzEWjGTaEQiHOf+3nhKyN/K12Mxn7XwXjDxqw8QtTChmXPo5l1b2n/IRLXPTWzGZi5kTa/e29NrOJRURTMM1H1uxsrKaz1jFqlGE+6pbA1r2fs2f9esRuJ/0HP6Dur/fR/vHHMc8VKyy1xddCgdu4kSekKYw3nc1RfgWH1UGoo32XNR0BWBwOXDNnDvU0IvRWEC/a0Dk8io9rNIPIpuY6lm5ez8raEr6q/Yba0Odc1upjZvoEOPz/Bvx8e+TswTe13/S6T4O7oUsfhVgUZ25ruBOO9EmE6HIVLZWbu5STtheNItTaSrYZ7tldU4iYj9atxzFxIiP+8Hu8ZWVU/ewmxr/4Ao7Ro7ucK1YCW6u3lZx2Y759RR9BVFhqWXmXm2iorX2XNR0NR3oTCirOe41mt+HKN/7Ml3Uf4qUGrJ2R9UoJc7zJXNm8FS59oV95CIkyOWsyb5W/FXG4xqLB3RDXyRwmHJZa1lLG/KL5CZ+/zl2Hw+Ig3ZFOXWUVrr32imwLC4iMBm9kHuFjIMp8tG4dKfPmYXG5GPW3+yg/7XQqr7mWcc8+06WvcmFyISvrVkaWw2WzM9uNW0tf0UeROdntXcJSAULt7btsOOpwpDfz0YywYxnYWzuaNbsbL6/+jCWNj6EIMDZpLgdmX8SPJ93OP2b/jWWjzuCxrWuxHfYrI0ltEJicZTRp39C0Ie4+8SqkRpOdlE2GM4Oylv4VxguHoxIK4d+ypaumYIalJtW2YBFLF/NRmiMNp9VJoKmJQF0dzilTACNyqejuP+Fdt46tt9zSxcfRPYGtM9BJUAVJbw0gDkekr0BviM2GY+wYvN0K44Xa27Huoolrw5HeQlKHRxcKjWaQuHvpX0Gl8O8zn6XAUwtrF8Pqv0GVaeefeCTMH7z6+1OyjJvpusZ1zCqY1WO7UooGT+wKqdGISMTZ3B/qOo0ktEB1NQQC2EdvEwoOU0AEq7aSlZIViYKqd9dH/AnedesBcE6eHDku9eCDybv2Gur+eh+uvWeQfd65wLaw1NrOWsakj6HFa9Q9SmnxYsvPT7g6qLN4At4NXYVosL0d+8jt6yeh6UmfTXY0mt2RJ79+nzbLak6z7EvBwqOgfp2xYeQ+hv9g6gmQN2W7E9QSIT85nwxnBuub1sfc3hnoxBv09qkpgGFCen/T+/06f21nLZOyJkUijxxRmoI1IwNLWpoRljozp4v5KCIU1hvzTpoyucu4OZdfjnvlKmruuIOkadNI3nefLs12xqSPiYS4OpvdCTmZwziKx9P2/vsovx+xG5nM2nw0sCTajlOj2W0IhUL87eu/YQ+k8Iuy18CZCsf9Ca5fDZd9CAf/DPKnDphAiBcqKiJMzpoc13wUq8RFvLHGZ4ynydtEk6cp4XmF6x75q8xw1G7OYfuoUfiqKsl15XZxNId7M3vWr8OaldWjhaRYLIy88w6sGRk0PfUU0DOrOVwMz97Y1i+h4CwuhkAA36ZNkXXafDSwaKGg+d7x4LL/4LaWcLU3SFJKgVHyev/LIGPgm7kH6uooOfgQWl5/Peb2KVlT2NC8gWAo2GNb9xIXbe+9x4a58/DX1PbYN9rZnAid/k7a/e1GH4XNm8FqxV7YNXLJMaooEpYazmoOm5zAMB85p0yJafqxpqWRPGsW7hUrgJ4JbGFNwdrQklDkUWROZlhquNyFUopgu44+Gki0UNB8rwiFQvxr1T9ICSRzQe16OOp34By8p8y6++4jUFdHy5tvxtw+OWsy7oCbyvbKHtvCmkLYfNTx+RcEm5tpeuaZHvuGw1ITFQpdO65VYh8xArF1tSbbi0YZlVKTsmnwNNDqa8UX8pHrykUFg3hLSnqYjqJxzZiBv7KSQENDjwS2Fm8LTp+Cjs6EIo/COMYbYanhchfK64VAQAuFAUQLBc33ins+eRmfdSM3tjVhHzUHpp/Zr+P9W7fS+u9/J5Q97FmzhuYXX8KSkkLnF0sJeb099glHIMXyK4RNNmHzUdjB2rxoESG3u8u+I1JG4LK5EnY2d+nNXFnZJfIojH3UKJTHQ6EnCW/QS0VrBbCtdadyu7s4mbvjmmFEbbm/NbWF5IJIAlurr5WsdmO//piPrKkp2AoKIglsoXZjEOsu2nVtOKKFguZ7gy8Q4Jn1D5Hjd3JKc41Rx6iXGv7d8dfWsvH8C6i64UaaX3yx132VUtTccSfWjAwKf/c7lMdD55c9s5cnZE7AIpZehUJWktF8xrt+Pc5JEwm2tNDy2mtd9rWIhXHp4yhvKe8xTiwimoIrH19lJY7RsYSCEZaabwQKsa7RcMbnunLxrA9HHk2Je46kPfcEqxX3im8BI4s72nyU22589v0RCgDOCcV4y43rDAsFrSkMHFooaL433PHxIgK2rfy8qRrbzPOgqGcYaDyCLS1svuRSAo2NJE2fTs3v/xC5Mcai/f336fziC3Kv+QlpRxyOOBx0fPxRj/2SbEmMTR8bueFG0+BuIMOZgd1iJ9DQQLCxkczTTydpr71oXPh4jwJ04zPGJ2w+CmsKOaQSbGjAPmp0j33C0UiZjT4A1jauBYwSF95160EE58QJcc9hcblwTpmM+1tDKBQkF3RxNI/0GAmB/RUKjvHF+MqMWk/BNlMopGihMFBooaD5XtDp9/JS2aOM9tk4OiBw5C0JHxvq7GTzFVfiKy9n9N//xuh/3I8lNZWqG27oYcYBUD4fNXfdhWPCBLLOOguLy0XyfvvR/vGSmONPzpocU1OITlzzrt+WE5B94YX4Kipo/6irkCnOKGZrx1Y6/Z09xupOvbseh8WBq9aMAjK1gmjsRca6tHpjvLDgykvOw7t+PY6xY7tkLcfCNWMGnhUrUcEghSmFNHmb8Aa9tPhaKOxH3aNoHMXjCbW3E6itI9ShzUcDjRYKmu8Fv3v/CUK2Om5u3Ir1kJshNbEbkfL5qPzpdbi//ZaRd99Nyrx52PLyKPrTXfhKy6j+wx96HNP49DP4N26i4Oc3R5y3qQcdiK+sDF9lVY/9p2RNoaq9inZfe5f10SUuIkJh0iTSjz0GW0EBjQsf77J/2Nlc3tq3CanWXWuUzK4y5tO9VhEYT/rW3Fyctc2A4fdw2Vyk2FPwrF/Xqz8hjGvvGYQ6OvCVlW3rq9BRQ6u3ldwOK5KcHLcPQzycUV3YtPlo4BkSoSAiFSKyUkS+EZFl5rpsEfmviGww/8bv4q3R9EEoFOKbrRXc++mrLHjlD7xV9RhTvYqDk4tgv8sTGkOFQmz55a/o+PhjCm+9hfRjjo5sS5k3j5zLLqPlxZdoeWNxZH2gqYn6f/yDlAMPJPXgg7ftf5DxvmNJzyqikXIXzV3zFaI1Bc+GDVizs7Hl5iJ2O1nnnUvn55/jWbs2sn8kLDUBZ3NdZ53pZN4MENPRDOAoKkK21iMInqCHPFceoc5O/Js24+wl8iiMa8YMANwrVnRJYDMczQp7Xl7C2cyRORVvC0uNmI+0UBgwhlJTOEwpNVMpNdtc/gXwnlJqEvCeuazRJExteytnv/BbDnjsDGYsnMv57/yQRzb8H8taniMzFOSW+hosx94Btr7r7iulqPn9H2h9803ybryBrDN7RinlXfMTXPvuS/Utt+CrqACg/m9/J9TZScHPb+6yr2P8OOxFRbR/FF8orG/sakKKLnHhXb8B56Rt/aCzzjwTcblofPyJyLoxaWOwirVPZ3MwFGRNwxomZk7EV1mJJTkZa1bsZzD7qFEEqqoizu5cVy7ekhJQiqQp8Z3MkeseNxZLejrub1d0SWBr8baQ0Rrst+kIDHOTJTkZX1m51hQGgeFkPjoJCOvDjwMnD91UNLsiP3njflZ3vkJAuRmTdABH5V/Fb2bez0d73sRHtZXsNfYwmHRU3ONVMIhnzRoan36ayquupumZZ8i++GJyLrkk5v5is1F0z92I3U7lDTfg+e47mp57jqyzzuxyAwcjeznl4IPo+PxzlM/XZVthSiFpjrQufgV/0E+br42cpBxUKIS3pKSLucaakUHmKafQungxgTqjcqndamd02ug+nc0lzSW0+duYVTDLyFEwu63Fwj5qFP6tW8m1G0IhLzkPzzrDt5CI+UgsFlzTp+P+9tsuCWytvlZSW/3bJRREBEex4WyO+BT6aYLSxGeoah8p4B0RUcCDSqmHgAKl1FZzezVQEOtAEbkMuAxgzJgxO2Ouml2AbzY3s6plCbmp4/jowjeMlY1l8OaNUPo+jNwXjr+nx3GetWtpe/c93F99hfvbbwl1dADG02jOpZeSd8P1vZo37CNGMOL//ZHKq65m43nnY0lOJveaa2Lum3rQQTQ/u4jOr74i5YADIuvD5S7WNW2LQAqHo2a7svFXVaE6O3FOmthlvOwLzqfp2WdpevZZ8q69FjC7sPUhFJbVGKGxswtm4658GHsv/0f2UUUQDDLWm8p6zMij9RuQ5OS4JqfuuGbMoP6f/8TpDZHhzGBr+1bavK24mvvvZA7jnFBMx9Ivce4xFXE6kV2069pwZKg0hQOVUvsCxwFXi8jB0RuVkRkUMztIKfWQUmq2Ump2Xl7i6fGa3ZdQSPHr1z/C6trEmdOOh4APPr4H/jEXNn9p1DW65F3I7OpM7Vy+nIozz6L+/vsJNDSQfuIPGfmnu5jw7rtM/N+H5N94Q0L27rTDDyf7wgsIdXaSe+WV2OKYYlL23x/s9rgmpA1NGwgpI8w0kriWlBNJWkvq9mTuGDeO1MMOo+nZRYQ8Rknq4sxiNrduxh/yx53v8prlFKUWUZhSaOQoxIg8ipzDvPGPbjOa7eS6cvGuW4dz0kQkwRwP14y9IRTCvWo1BckFlDSXkORVWL2B7RYKjvHFBLZuJVBbp01HA8yQaApKqSrzb62IvALsB9SIyAil1FYRGQH0LPCi0cTg+WWbWd/+GUmpcEJyITx4MNStgT1ONBLU0nuWVfaWlbP5qquxFxUx9sknsHUr6tZf8n/2M1IOPIiUeXPj7mNJSSF51iw6Pv4Ybr6py7YpWVPoDHRS1V7F6LTRNLqNukfZSdl4138OgGPipB5jZi+4kE3vv0/L66+TdeaZFGcUE1ABNrdujkQjRaOUYnnNcg4sOpBgYyPK7Y6ZoxAmrA0UtgrkQZ4pFNKOPjruMd1J2tvMbF7xLYXFhXxd8zXZkWzm7Xuwc5hd2DwrV2LVQmFA2emagoikiEha+D1wNLAKeB240NztQuC12CNoNNto7vRx53/WkpW3hslJeYxbdCH42uGcRXDWkzEFQqC+ns2XXYbYbIx+6MEdFggAYreTetCBiLX3NiSpBx2Ed8MG/Fu3dlnfvdxFdIkL7/r12IuKIv2To0meMwfntD1ofPwJlFJ9FsYrby2n0dNo+hPCkUfxNQV7YSFYLOQ0GQX78jptBFtaEvInhLFlZWEfO8bwKyQX0OZvI6sfHddiEQlLrajQmsIAMxTmowJgiYh8CywF3lRK/Qe4AzhKRDYAR5rLGk2v3P3OOtoCDbitpRxdXQrFh8FVn8OU42LuH+rsZPOVVxGor2f0Px+IGZ8/mKQcdCAA7Uu6JrJNyJyAIJEIpOgKqd4NG+LehEWEnAsvxFdaSucXXzA+w3iCjicUltcsB2BWwaxtfRR6+QzEbsdeWEhmo1G3KW+L4XNJJBw1GteMGYZQcBlCIKvNWG/bThOwfcwYMAWwFgoDy04XCkqpMqXUDPO1p1LqD+b6BqXUEUqpSUqpI5VSjTt7bppdi1VVLTz9xSaOnGIUXDvaXgBnPm70R4iBCgapuvFneFavpujP9+CaPn1nThcwks9shYV0dPMrJNuTGZs+dpum4DYqi7qUDW95RY9opmjSjj4aMX0VyfZkClMKexUKua5cxqSN2dZHoSi+pgCGCSmvSfG3w/9GVpVxN+/u3+gL194zCNbVM6rTyICOmI+2UyhYHI6Iv8Ois5kHlOEUkqrRJEwopPjta6uYlNyJ272YiYEQ43/0IiTF7vWrlKLmD3+g/YMPKPjNr0k7/PCdPGMDESH1oAPp+OwzlL+rM3hS1qSIUAgnrnnLKyAQ6NVcY3G5cM2eRccnnwDEbc2plGJZ9TJmFcxCRPBt3ow1L7fPUhXhXIVDRx+KZ906bAUFWDMz+3Xd4YqpBRuNshpZ7QpSUvqdzRyNY4JRd8mq6x4NKFooaHZJXvqqktWbavl7+r18bVMcPfk0yIwdWqlCIRoe/hdNzzxLziU/JvtHP9rJs+1KykEHEWpvx/3NN13WT86azOa2zXT6OyMlLrbVPIqvKQCkzp+Pd906/LW1FGcUU9FaEYlkCrOlYws1nTWRftD+yiocvTiZw9hHFRGoqyPk8ZiNdfqnJQAkTZmCOBxkbDAK4mW1g307ncxhnKazWZuPBhbdo1kzbPH6/dS2e7BZuv5MPf4Qd731HY9m/IvlgY0oyeLo6Rd22Sfk89H5xRe0vfsebe+/R7CunvQfHEfeDTfszEuIScrcuWCz0f7xEpLnzImsn5I1BYViQ/MGGjwNjEwdifebDWCz4Rw3rvcx58+Hu++h87PPGD91PO6Am+qOakambnO0L6velp8A4N+8GdesvivFhs00vo2b8JaVkWr6RfqDOBwkTZtGaE0ZjIWcdsE+KmYqUsKEu7Bp89HAooWCZliyrm4LZ79+MV6/0LnxclD2Lttvti1ifnAJ/5qwPxMcyUzInIBSira336btnXdo/99HhDo6sCQnk3LwwaQdcQTpxx6TcGz9YGJNSyN55kzaP/6Y/Buuj6yfnL0tAqnR08j03OlGD4Xx4/tMznJOmYI1J4f2JZ9QvL9RkqOspayLUFhes5wMZ4bxWfn9+KuryYjRR6E74bDUjiVLwO/HmUB5i1i4ZsygadEisk9IJ7u9ebtzFMKEw1J1SOrAooWCZtjxXU0V5y5egLLUYHUpzh/3O67vcEa2iwqS1bae+n1+xPLmT7h8slHgrvm556m+9VasOTmk/+A40o48kuQDDsDidMY71ZCRctBB1P3lL/hrayNhmSNTRpJqT2Vtw1qaPE1mjsLHuGbO7HM8sVhImTePjk8/ZfzvjLpLZc1lHFi07al+ec1y9s3fF4tY8G2thFAIe1ECQsHcp+3994HEylvEwjVjbxoff5y9m0eS2da03eGoYZyTJmPNy8UxcWLfO2sSZugfmzSaKFZsqeS8Ny7AYqnmnzXVXEgGryYFWJ6fTXbhOLILx5E1YgLMu4b3Jh+EQnH02KNRwSANjz5K0vTpTProf4y4/XZSDzlkWAoEgNSDDwKgY8knkXXhchdf1nxJUAXJU6n4t2xJ+CacMn8ewYYGUjbWkenM7BKBVNtZy6a2TVH+BDPyKAFNwZaXizgcuL/+2jBlmX2S+0u4YuohlanYgmqHNQVragqTP/6YtEMP3aFxNF3RQkEzbFi+eTM/fvM8LNYa/l7XxAEnPMBPz/uA6bnTucXSTOUP74YfLTJeR/+e/256n3Hp45iYOZG2/76Lf9Mmcn784z4TyIYDzqlTseXl0fbuu13WT8qaFKlyWlBjFM5LWCjMmwdAxyefUJxR3KVa6lc1XwHb/AmRHIUE6heJxWKErYZCOIuLt7vOkG3kSKy5uey7NmAs76BQ0AwOWihohgVfbNzINf85G2Wr597WEHPPWwx7nYbdaueug+8C4OaPbsYfNMI4G9wNfFnzJUePM8otNDzyCPYxY0g76sghu4b+ICJknHoq7R98gG/jxsj6cGYzQFaVEb7ZV+RRGHt+Ps7Jk2n/5JMerTmX1Swj2ZbMlGzDH+CvrAS7HVtBYs7esF9he01HYFyza8YMvGvWAFooDFe0UNBsF26/j++qa6mo79jh13++W8/P3zkDv62ZP/uzOPDHH8LImZFzjUobxa3zbmVl/Uru+/o+AN7f/D4hFeLosUfT+eWXeFauJOfii3YJLSFM1rk/Qmw2Gh/f1kEtfNMGSN5UjyU5GfvInqU64pEyfz7uZcuZmDSaZm9zJDN6ec1y9snfJxLJ5avcjH3kiIQ/r3ApjO0JR43GZdZBAi0Uhiva0azpFy1uH3e8/xwfbL2fgMXDiVXjKPQmJ3SsoLATwIkfh/hxEqDT6uGxokY6bCHuStqbg89fCLaefoCjxx3NWdVnsXD1QuYUzuGdincYmz7WiO1/5Aqs2dlknHzywF7sIGPPzyf9xB/S/PIr5F5zDbasLCZlbtMKbBVVWPpRjRQModD42GNM2mhoVGXNZUimUNJcwvHFxwNGuK77629Imjo14XHDZqb+ZjJ3J5zEBtufzawZXLRQ0CTElmY3T731JEuaH6U8uYNxyk8gJLw5agP31bUyyxu/VHM0IYudkMVB0Opkvd3OjRkhOizwp5Enc+hRv4eoUtXesnLEIjjMGP2b5tzE17Vf8+slv6bN18bFe12Md8MGOv73EbnXXoMlKWkwLn1QyVmwgJaXXjZ6Ilx1Fcn2ZEanjWZLWxXBknKSjzyiX+Mlz56FOJ3krqyEsUZYaouvBSDiZG5+7nkC1dVk/eH3iY+73344xo+PVDzdXpL2mg4iWNPTh20QwPcdLRR2Aeo72nH7VI8kLoBAUOENhPD4g3j9QfzuVppbN7GlvYQxSWNJsuzYjVKFgmxa8Sbf+v7NOxkKV5LiGhnNRYfdRFPhnlz67uX8xL6Few+7l/lF8xMe9/Otn3P9B9eTbE/miSP+0cVsEmxro+5vf6PpqaexJCczZuFCXHvtidPq5E+H/ImzF59NUAU5auxRNP7pMcTlIuucc3boOocK56RJpBxyME1PPU3Oj3+MxelkavZUHM2dBJtqeq15FAtLUhLJs2fj/+IbXBNclLeUU9FagdPqZM+cPQl1dlL/4IMkz5kTcUwngmv6dCa89e/+Xl4PrKkpOCdOJE67FM0wQAuFBAkEQ3gDob533JFzhBSbGjopraqmZeMKAjVroPU7nixYQ6clxMxOC7M6LEzvtOBUxhO1Az/pdJJk6eTLFMV/UpP5wpVEUASbUszweJnr8XCA28OeXl9CX7gCaq1W1jnsrHI6eTY9lVaXlRPS9uTGw+8gO8sIScwHHjv2Ma747xX85P2fcPfBd3PE2L6fbN8ofYPffvJbxmWM44EjH4g0dFdK0bp4MTV33UWwvoHM00+n45NP2HzJJYx96kmcEydSnFHM/zvw//FR1UdM8GVSungxWeecE7exza5AzkUXs2nBAlpee42sM8/kp/v+lIb2D4A7t8uxmzJ/PrV33cXeoamUtZTR7G1m77y9cVgd1D/9OMH6evLu+2tCDYQGg7wbrkf5EtMsNTsfMZqc7ZrMnj1bLVu2bFDPsbXFzZ3vPs6nzY9gDwnFLfmMb83HFdwWlmcnQLJ4cWG8ksVL0OKmw+4j128hOZTYP5+VEGMtNYySegACwKWFBXyb5OTggIulNg9tonAomB1I4sCAC6fY+MDh4wtLK34UBZYUDkmewh4p4/jOU8XX7gpKfTUoIMXiZK+kUWRaU0gSO06LHafYSBI7drFRE2im1FdLmbeW1pA7Mq9ZmdP4xUG/Y2q2YYNWoRBt77yDv6qKjFNPpTPFylXvXsWq+lXcPv92fjjhhzGvTynFwysf5m9f/439C/fnL4f9hTRHGgDeDRuovu12Or/8kqTp0yn87W9xTd8L38aNVJx3HoIw9pmnu5R5rrnzLhqfeIIJb7/da/ew4Y5SiorTTifkdlP85mLEYqFh4UJq77iTSZ9+gi07u1/jedatp/ykk/hkwUyeGFdFi6+Fy/a+jCsmXEDJkUfhmrE3Yx56aJCuRrOLEPempDWFOKwoq2LJfx/lc/+rfJ3mY0rQR7IK8XWem1W5FRzW6ebUtnbmuj1YgTaLnS+TU/kiKYkvk+yU2MB8mCc9JIwKWigKWhgVsjAuYOUAvw1rj+/FSjBjfxpG7knmmL25p+EzlpW9zt/bfsgevlzsUydTUqB417+S9yrf59POGsDoznXGuHP4QfEP2NMxDv+mzYTa2zhl7jSsGRk0eZr4ovoLPt/yOV/Xfk25rxp3wI076CYQCkTO7rA4mJQ1iaNGzWZK9hSmZk9lctZkUuxGJUulFB2ffErtn+/B+50RVlh3/z/IOvtsHjj/j1y/4jZ+veTXNHoamZw1GV/QhzfojbyW1yxncdliTig+gdvm3YbdakcFAtT99T4aHnsMS0oKhbfeSuYZp0eiYhxjxzLmkUfYdP4FbFpwEWOfeRp7QQHB1laan3+e9GOP3aUFAhihmtkXX8yWn/2M9g//R9rhh+HdsAFrTk6/BQIYIazWvFwmrXfTNKIJMPwJjY8tJNTSQt5PfzrQl6DZjfheagpry77io5WvMNE+kkxr17opPncbgTWLabSu5J7cNFotFi5yTeHSqZfjHDuLCn81L5e9zuvl/6bJ20xhcgGFyYWsalhNQAWwW+wc317Msf/rIH1zE3XzJvP1/HxWuxrZ2LqR2k6jy+j8ovncedCdZDhjl3p+o/QNfvXxL/l/X09hwtvfgcUCIcN8ZUlJwTl1Kh3j8wmkOMlpDBDYtBnfpk0Em5q6jOOcNAnXvvuSPGtfXPvui72oqIvZwB/y4wl48AQ8ZCVlxfRbALhXrqT2nj/T+fnn2EeOJO+n15I0bRr1Dz1M65tvInY7aaefyl/32MhbHUtjjiEIl0y/hGv2uQYRIdjSQtX1N9Dx6adknHIK+Tf9LO5N0L1yJZsuXICtsJCxTz1J80svUXfPnxn/8kskTZsW85hdCeX3U3L0MThGjWLsk09QfsaZWFJTGPvYY9s13paf/4LGD97lR1d5sFrsfHTsYrYceyIpBx7IqPv+OsCz1+yCxNUUvpdC4Z8Lr2HphvepyBdsKUH29nqZ4fWxt9dLfiDI73Py+SjFznzPCK53H4pjydd4Vq1CXC7SDj+c9BOOx3nA/vyv9hNe3vAyrd5W5hTMZv7WNHKe+xDvsuVYs7Jw7b037Z98AoEAKQceSNaPzsEybw5vVLzJnV/eyYiUEfz1sL8yKaurM3F1/Wou+Pf53PBxBvt+XE3WBeeTf+ONeEtK8K5Zg+e7NXi++w7PunUojwfbiEIcY8biGDMGx9ixOMaOQVwuPCtW0PnV17i//ppQu9HVxJqXi3N8MfbRo3CMHoNjzGjso0djHzUKsVoJuT0or8f463ETbG+n+YUXafvPf7BmZZF75RVknn02lqisVl9FBfUPPUzL668bdfqPO5jgeSfiGFmEw+ogyZqEw+og2Z4cMRf5KirYfMWV+KqqGHHLb8k8/fQ+v7eOpUvZfOllOIqLCdbX45w0iTGPPtLv73+40vDYQmrvvJNxzz/HxgsXkHnG6RT+6lfbNVbLG2+w5aab+fkCK+l7z+SeVTNoXLiQ4tdfMx29mu85WihEs/npx2m/3ej26U2ysjnfQklugI0FQnUm7FVp5diNGaRsMmz7SXvvTdoRR+DfuoW2t/5DsKUFa0YGacceS8YJxxNsa6P+nw/iWbECW34+OT++mMwzzsCSnIy/ppbmF18wwgBra7GNGEHWWWdSecSeXP/1b+nwd3D7/Ns5ZtwxgJGpe/brZ3Lmmy0cuLSD7AULyP/5zTGdgioYRAWDXW7QsVDBIN6SEjqXL8fz7Qp8mzbhq9xMsK4+oc9LkpPJWbCA7Isv6rUipa+yioaHH6b55ZcRIPOM08m5/HLs3bJmOz7/nMqfXoeIMOpv93UpH90X7R99xOarfwJ+P6Mf+Rep8xOPeBruBNvbKTn0MBzFxXhWrGDE729PSFjGIlBfz4YDD+Klw5MoOP1sDrr+WdKPPZaRd+outxpAC4WuhNxuvOvX41m7Du+6tXjWrcezdg2qo9PYQQTXrH1JP/po0o46CvuIEZFjlc9H+6ef0rr4Tdreew/lNhyy9qIici69lIxTT4l5k1Z+P20ffEDTs8/S+dnniMuF85QTuGvCWpYE1nDxXhdz1cyruPzty9jvia847JsAOZdeQt4NNwxalEiosxPf5kr8lZvxV1ailMKS5MLiSkLCf51JOCdP6ld0j3/LFuoffIjml15CLBYyzzqLnEsvwZ6fT9OiRVTf/nsc48cx+oHt65Hc9uGHuJd/Rd4N1w9ZBM1gUfOnP9H4yKMAjHv+uS4ZwP2l7JRTCSY7SZ44mdaXXmbCW//e6T2pNcMWLRT6QimFv6oKX3k5SWaxsr4IdXbS9sEHiAhpRx2F2O19HgPgWb+exkceoWXxmyBC+QGjuW+PTQSK8jn1pWoOW6nIueJy8n760136puerrKLhwX/S/PIriM1G8qx96fj0M1IOOZiie+7RdfBj4K+upuTIoyAQYMryZTvUrrL27rtpWPg4iJB52qmMuPXWgZuoZldHC4XhiL+qioaFj9P8wgsoj4eqXKGoXpF79dXk/uTqXVogROPbvJn6B/5Jy+uvk33eeeTf9LNdqkbRzmbrLbfiWbmS8S+/tEPjdHz2GZsuuhhxOpnwzts9zHia7zVaKAxnAk1NND35FE0vvkD2j84l94rLh3pKg0LI5+vT/6ExfEAohdh2LGI85PVSctjhZJ52Gvk3Dn0bUs2wQgsFjeb7SKijA3G5hkUbUs2wQievaTTfR3bEJ6H5fqIfHzQajUYTQQsFjUaj0UTQQkGj0Wg0EbRQ0Gg0Gk0ELRQ0Go1GE2GXDkkVkTpgYx+75QKJFfnZvdDX/f3j+3rt+rr7T71S6thYG3ZpoZAIIrJMKTV7qOexs9HX/f3j+3rt+roHFm0+0mg0Gk0ELRQ0Go1GE+H7IBS+r81o9XV///i+Xru+7gFkt/cpaDQajSZxvg+agkaj0WgSZLcWCiJyrIisE5ESEfnFUM9nsBCRR0WkVkRWRa3LFpH/isgG82/irdN2EURktIh8ICLfichqEfmpuX63vnYRSRKRpSLyrXndvzPXjxeRL8zf+3MislvWKRcRq4h8LSKLzeXd/rpFpEJEVorINyKyzFw3KL/z3VYoiIgVuB84DpgGnCMi04Z2VoPGQqB7zPEvgPeUUpOA98zl3Y0AcKNSahpwAHC1+R3v7tfuBQ5XSs0AZgLHisgBwJ3AX5RSE4Em4MdDN8VB5afAmqjl78t1H6aUmhkVhjoov/PdVigA+wElSqkypZQPWAScNMRzGhSUUh8Bjd1WnwQ8br5/HDh5Z85pZ6CU2qqU+sp834ZxoyhiN792ZdBuLtrNlwIOB1401+921w0gIqOA44F/mcvC9+C64zAov/PdWSgUAZujlivNdd8XCpRSW8331cBu3YtRRMYB+wBf8D24dtOE8g1QC/wXKAWalVIBc5fd9fd+L3AzEDKXc/h+XLcC3hGR5SJymbluUH7nusnO9wCllBKR3TbMTERSgZeA65RSrdG9rXfXa1dKBYGZIpIJvAJMHdoZDT4icgJQq5RaLiKHDvF0djYHKqWqRCQf+K+IrI3eOJC/891ZU6gCRkctjzLXfV+oEZERAObf2iGez6AgInYMgfC0Uuplc/X34toBlFLNwAfAXCBTRMIPervj730+cKKIVGCYgw8H/sruf90oparMv7UYDwH7MUi/891ZKHwJTDIjExzA2cDrQzynncnrwIXm+wuB14ZwLoOCaU9+BFijlPpz1Kbd+tpFJM/UEBARF3AUhj/lA+B0c7fd7rqVUr9USo1SSo3D+H9+Xyl1Lrv5dYtIioikhd8DRwOrGKTf+W6dvCYiP8CwQVqBR5VSfxjaGQ0OIvIscChG1cQa4BbgVeB5YAxGJdkzlVLdndG7NCJyIPAxsJJtNuZfYfgVdttrF5G9MRyLVowHu+eVUreJSDHGE3Q28DVwnlLKO3QzHTxM89HPlFIn7O7XbV7fK+aiDXhGKfUHEclhEH7nu7VQ0Gg0Gk3/2J3NRxqNRqPpJ1ooaDQajSaCFgoajUajiaCFgkaj0WgiaKGg0Wg0mghaKGh2OiKiROSeqOWficitAzT2QhE5ve89d/g8Z4jIGhH5IMa2SSKyWERKzbIEH4jIwYM9p3iIyMnRxSBF5DYROXKo5qMZ3mihoBkKvMCpIpI71BOJJiorNhF+DFyqlDqs2xhJwJvAQ0qpCUqpWcA1QPHAzbQnZlXgeJyMUSkYAKXUb5VS7w7mfDS7LlooaIaCAEYrweu7b+j+pC8i7ebfQ0XkfyLymoiUicgdInKu2VdgpYhMiBrmSBFZJiLrzXo54QJyfxKRL0VkhYhcHjXuxyLyOvBdjPmcY46/SkTuNNf9FjgQeERE/tTtkHOBz5RSkex5pdQqpdRC89gUMfpfLDV7Apxkrl8gIi+LyH/M+vh3Rc3haBH5TES+EpEXzFpP4Rr7d4rIV8AZInKpeX3fishLIpIsIvOAE4E/iVGLf0L0ZywiR5jzWGnOyxk19u/Mc64Ukanm+kPMcb4xj0vr68vW7FpooaAZKu4HzhWRjH4cMwO4AtgDOB+YrJTaD6OM8jVR+43DqA1zPPBP8+n9x0CLUmoOMAe4VETGm/vvC/xUKTU5+mQiMhKjVv/hGH0L5ojIyUqp24BlwLlKqZu6zXFP4KteruHXGOUZ9gMOw7hZp5jbZgJnAdOBs8RoIpQL/AY4Uim1r3neG6LGa1BK7auUWgS8rJSaY/ZZWAP8WCn1KUY5hJvMWvylUdeXhNGL4yyl1HSMbNkro8auN8/5APAzc93PgKuVUjOBgwB3L9eq2QXRQkEzJCilWoEngGv7cdiXZg8FL0ap6HfM9SsxBEGY55VSIaXUBqAMo4Lo0cAFYpSb/gKj5PIkc/+lSqnyGOebA3yolKozSzM/DfTLNyAir5haRrhY39HAL8x5fAgkYZQpAKNhSotSyoOhtYzFaB40DfjEPOZCc32Y56Le72VqPSsxNJY9+5jeFKBcKbXeXH682/WF57ycbZ/vJ8CfReRaIDOqZLVmN0GXztYMJfdiPFU/FrUugPmwIiIWILq1YnQ9m1DUcoiuv+XutVsUIMA1Sqm3ozeYNXQ6tmfycVhN1I1VKXWKiMwG7g6fEjhNKbWu2zz2p+v1BTGuSYD/KqXOiXO+6LkvBE5WSn0rIgsw6mHtCOH5hOeCUuoOEXkT+AGGoDpGKbU23gCaXQ+tKWiGDLN41/N0bZ9YAcwy35+I0VWsv5whIhbTz1AMrAPeBq4Uo9Q2IjI5ymwTj6XAISKSazpyzwH+18cxzwDzReTEqHXJUe/fBq4RMZo+iMg+fYz3uTneRHP/FBGZHGffNGCreY3nRq1vM7d1Zx0wLjw2hkmu1+sTkQlKqZVKqTsxKhHv9n0cvm9ooaAZau7BqO4a5mGMG/G3GD0CtucpfhPGDf0t4ArTHPMvDJPMVyKyCniQPjRls6vVLzBKM38LLFdK9VqeWCnlBk4ArjAd4p9h+AR+b+5yO4agWyEiq83l3sarAxYAz4rICuAz4t+I/w/DNPYJEP30vgi4yXQMRxzy5udyEfCCaXIKAf/sbT7AdaY5bAXgx/iMNbsRukqqRqPRaCJoTUGj0Wg0EbRQ0Gg0Gk0ELRQ0Go1GE0ELBY1Go9FE0EJBo9FoNBG0UNBoNBpNBC0UNBqNRhNBCwWNRqPRRPj/ytmJl+peabYAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "num_generations = 50\n", + "num_rollouts = 20\n", + "print_every_k_gens = 5\n", + "\n", + "rng = jax.random.PRNGKey(0)\n", + "es_logging = ESLog(param_reshaper.total_params,\n", + " num_generations,\n", + " top_k=5,\n", + " maximize=True)\n", + "\n", + "# No es_params!\n", + "state = strategy.initialize(rng)\n", + "\n", + "for gen in range(num_generations):\n", + " rng, rng_init, rng_ask, rng_eval = jax.random.split(rng, 4)\n", + " x, state = strategy.ask(rng_ask, state)\n", + " fitness = evaluator.rollout(rng_eval, x).mean(axis=1)\n", + " state = strategy.tell(x, fitness, state)\n", + " if gen % print_every_k_gens == 0:\n", + " print(\"Generation: \", gen, \"Performance: \", state.best_fitness)\n", + " #break\n", + " \n", + "es_logging.plot(log, \"CartPole Augmented Random Search\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fc13efd2", + "metadata": {}, "outputs": [], "source": [] } diff --git a/examples/01_classic_benchmark.ipynb b/examples/01_classic_benchmark.ipynb index 6912e2d..1fd1890 100755 --- a/examples/01_classic_benchmark.ipynb +++ b/examples/01_classic_benchmark.ipynb @@ -33,26 +33,18 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 3, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - ":228: RuntimeWarning: scipy._lib.messagestream.MessageStream size changed, may indicate binary incompatibility. Expected 56 from C header, got 64 from PyObject\n", - "WARNING:absl:No GPU/TPU found, falling back to CPU. (Set TF_CPP_MIN_LOG_LEVEL=0 and rerun for more info.)\n" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "CMA-ES - # Gen: 10|Fitness: 0.13|Params: [0.6441135 0.41466928]\n", - "CMA-ES - # Gen: 20|Fitness: 0.00|Params: [0.97413015 0.9518173 ]\n", - "CMA-ES - # Gen: 30|Fitness: 0.00|Params: [0.9981632 0.9965331]\n", - "CMA-ES - # Gen: 40|Fitness: 0.00|Params: [0.9999719 0.9999461]\n", - "CMA-ES - # Gen: 50|Fitness: 0.00|Params: [0.9999997 0.9999994]\n" + "CMA-ES - # Gen: 10|Fitness: 0.11798|Params: [-0.24922156 -0.45996755]\n", + "CMA-ES - # Gen: 20|Fitness: 0.06408|Params: [-0.25254983 -0.44303334]\n", + "CMA-ES - # Gen: 30|Fitness: 0.00020|Params: [-0.00756136 -0.01385564]\n", + "CMA-ES - # Gen: 40|Fitness: 0.00000|Params: [-0.00087966 -0.00171674]\n", + "CMA-ES - # Gen: 50|Fitness: 0.00000|Params: [-3.9389306e-06 -8.1345934e-06]\n" ] } ], @@ -60,25 +52,27 @@ "import jax\n", "import jax.numpy as jnp\n", "from evosax import CMA_ES\n", - "from evosax.problems import ClassicFitness\n", + "from evosax.problems import BBOBFitness\n", "\n", "# Instantiate the problem evaluator\n", - "rosenbrock = ClassicFitness(\"rosenbrock\", num_dims=2)\n", + "rosenbrock = BBOBFitness(\"RosenbrockOriginal\", num_dims=2)\n", "\n", "# Instantiate the search strategy\n", "rng = jax.random.PRNGKey(0)\n", "strategy = CMA_ES(popsize=20, num_dims=2, elite_ratio=0.5)\n", - "state = strategy.initialize(rng)\n", + "es_params = strategy.default_params.replace(init_min=-2, init_max=2)\n", + "\n", + "state = strategy.initialize(rng, es_params)\n", "\n", "# Run ask-eval-tell loop - NOTE: By default minimization\n", "for t in range(50):\n", " rng, rng_gen, rng_eval = jax.random.split(rng, 3)\n", - " x, state = strategy.ask(rng_gen, state)\n", + " x, state = strategy.ask(rng_gen, state, es_params)\n", " fitness = rosenbrock.rollout(rng_eval, x)\n", - " state = strategy.tell(x, fitness, state)\n", + " state = strategy.tell(x, fitness, state, es_params)\n", "\n", " if (t + 1) % 10 == 0:\n", - " print(\"CMA-ES - # Gen: {}|Fitness: {:.2f}|Params: {}\".format(\n", + " print(\"CMA-ES - # Gen: {}|Fitness: {:.5f}|Params: {}\".format(\n", " t+1, state.best_fitness, state.best_member))" ] }, @@ -91,96 +85,110 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "SimpleES - # Gen: 5|Fitness: 0.44|Params: [0.41951638 0.14410228]\n", - "SimpleES - # Gen: 10|Fitness: 0.04|Params: [0.91297907 0.8160112 ]\n", - "SimpleES - # Gen: 15|Fitness: 0.01|Params: [0.98460174 0.97844493]\n", - "SimpleES - # Gen: 20|Fitness: 0.01|Params: [0.98460174 0.97844493]\n", - "SimpleES - # Gen: 25|Fitness: 0.01|Params: [0.98460174 0.97844493]\n", - "SimpleES - # Gen: 30|Fitness: 0.01|Params: [0.98460174 0.97844493]\n", + "SimpleES - # Gen: 5|Fitness: 2.41|Params: [0.5749427 1.3363407]\n", + "SimpleES - # Gen: 10|Fitness: 0.05|Params: [-0.02394086 -0.06951416]\n", + "SimpleES - # Gen: 15|Fitness: 0.02|Params: [0.052019 0.11855024]\n", + "SimpleES - # Gen: 20|Fitness: 0.02|Params: [0.052019 0.11855024]\n", + "SimpleES - # Gen: 25|Fitness: 0.02|Params: [0.052019 0.11855024]\n", + "SimpleES - # Gen: 30|Fitness: 0.02|Params: [0.052019 0.11855024]\n", "====================\n", - "SimpleGA - # Gen: 5|Fitness: 6.79|Params: [-0.012256 -0.24003565]\n", - "SimpleGA - # Gen: 10|Fitness: 0.68|Params: [0.21533592 0.02063736]\n", - "SimpleGA - # Gen: 15|Fitness: 0.39|Params: [0.4103716 0.14900509]\n", - "SimpleGA - # Gen: 20|Fitness: 0.18|Params: [0.5903524 0.33600026]\n", - "SimpleGA - # Gen: 25|Fitness: 0.17|Params: [0.6199676 0.39935672]\n", - "SimpleGA - # Gen: 30|Fitness: 0.13|Params: [0.64335036 0.40990546]\n", + "SimpleGA - # Gen: 5|Fitness: 0.05|Params: [-0.23124489 -0.40957353]\n", + "SimpleGA - # Gen: 10|Fitness: 0.03|Params: [-0.13031456 -0.23322381]\n", + "SimpleGA - # Gen: 15|Fitness: 0.02|Params: [-0.11532619 -0.20849138]\n", + "SimpleGA - # Gen: 20|Fitness: 0.00|Params: [-0.00291448 -0.00076399]\n", + "SimpleGA - # Gen: 25|Fitness: 0.00|Params: [-0.00291448 -0.00076399]\n", + "SimpleGA - # Gen: 30|Fitness: 0.00|Params: [0.0479427 0.09704389]\n", "====================\n", - "PSO - # Gen: 5|Fitness: 1.11|Params: [-0.01428866 0.02790421]\n", - "PSO - # Gen: 10|Fitness: 0.03|Params: [1.0889671 1.1718146]\n", - "PSO - # Gen: 15|Fitness: 0.01|Params: [1.109518 1.2260276]\n", - "PSO - # Gen: 20|Fitness: 0.01|Params: [1.07492 1.1620886]\n", - "PSO - # Gen: 25|Fitness: 0.01|Params: [1.07492 1.1620886]\n", - "PSO - # Gen: 30|Fitness: 0.01|Params: [1.07492 1.1620886]\n", + "PSO - # Gen: 5|Fitness: 0.32|Params: [-0.01428866 0.02790421]\n", + "PSO - # Gen: 10|Fitness: 0.19|Params: [-0.32952115 -0.5220759 ]\n", + "PSO - # Gen: 15|Fitness: 0.12|Params: [-0.33950207 -0.56108034]\n", + "PSO - # Gen: 20|Fitness: 0.09|Params: [-0.30322644 -0.5158702 ]\n", + "PSO - # Gen: 25|Fitness: 0.06|Params: [-0.23692334 -0.41880134]\n", + "PSO - # Gen: 30|Fitness: 0.06|Params: [-0.23692334 -0.41880134]\n", "====================\n", - "DE - # Gen: 5|Fitness: 0.33|Params: [0.4517086 0.18653232]\n", - "DE - # Gen: 10|Fitness: 0.06|Params: [0.7975259 0.6230468]\n", - "DE - # Gen: 15|Fitness: 0.00|Params: [0.95278853 0.90621567]\n", - "DE - # Gen: 20|Fitness: 0.00|Params: [0.9835618 0.9694447]\n", - "DE - # Gen: 25|Fitness: 0.00|Params: [1.0125908 1.0251266]\n", - "DE - # Gen: 30|Fitness: 0.00|Params: [1.0005985 1.0011392]\n", + "DE - # Gen: 5|Fitness: 0.37|Params: [-0.6068972 -0.8382896]\n", + "DE - # Gen: 10|Fitness: 0.03|Params: [-0.15622607 -0.2968421 ]\n", + "DE - # Gen: 15|Fitness: 0.00|Params: [-0.01819804 -0.03231879]\n", + "DE - # Gen: 20|Fitness: 0.00|Params: [0.0003629 0.00136572]\n", + "DE - # Gen: 25|Fitness: 0.00|Params: [0.0003629 0.00136572]\n", + "DE - # Gen: 30|Fitness: 0.00|Params: [0.00027412 0.00099771]\n", "====================\n", - "Sep_CMA_ES - # Gen: 5|Fitness: 5.25|Params: [-1.2778556 1.6572908]\n", - "Sep_CMA_ES - # Gen: 10|Fitness: 5.25|Params: [-1.2778556 1.6572908]\n", - "Sep_CMA_ES - # Gen: 15|Fitness: 5.25|Params: [-1.2778556 1.6572908]\n", - "Sep_CMA_ES - # Gen: 20|Fitness: 5.25|Params: [-1.2778556 1.6572908]\n", - "Sep_CMA_ES - # Gen: 25|Fitness: 5.25|Params: [-1.2778556 1.6572908]\n", - "Sep_CMA_ES - # Gen: 30|Fitness: 5.25|Params: [-1.2778556 1.6572908]\n", + "Sep_CMA_ES - # Gen: 5|Fitness: 4.09|Params: [-1.5836709 -0.5336474]\n", + "Sep_CMA_ES - # Gen: 10|Fitness: 4.09|Params: [-1.5836709 -0.5336474]\n", + "Sep_CMA_ES - # Gen: 15|Fitness: 4.09|Params: [-1.5836709 -0.5336474]\n", + "Sep_CMA_ES - # Gen: 20|Fitness: 4.09|Params: [-1.5836709 -0.5336474]\n", + "Sep_CMA_ES - # Gen: 25|Fitness: 4.09|Params: [-1.5836709 -0.5336474]\n", + "Sep_CMA_ES - # Gen: 30|Fitness: 4.09|Params: [-1.5836709 -0.5336474]\n", "====================\n", - "Full_iAMaLGaM - # Gen: 5|Fitness: 0.25|Params: [0.6604285 0.39947018]\n", - "Full_iAMaLGaM - # Gen: 10|Fitness: 0.13|Params: [0.69210684 0.4968851 ]\n", - "Full_iAMaLGaM - # Gen: 15|Fitness: 0.04|Params: [0.7911089 0.6271153]\n", - "Full_iAMaLGaM - # Gen: 20|Fitness: 0.01|Params: [0.8877124 0.78315926]\n", - "Full_iAMaLGaM - # Gen: 25|Fitness: 0.00|Params: [0.97602683 0.9512631 ]\n", - "Full_iAMaLGaM - # Gen: 30|Fitness: 0.00|Params: [0.99968135 0.999422 ]\n", + "Full_iAMaLGaM - # Gen: 5|Fitness: 0.01|Params: [-0.10261801 -0.19658661]\n", + "Full_iAMaLGaM - # Gen: 10|Fitness: 0.00|Params: [-0.0065736 -0.01341229]\n", + "Full_iAMaLGaM - # Gen: 15|Fitness: 0.00|Params: [9.990766e-05 1.880897e-04]\n", + "Full_iAMaLGaM - # Gen: 20|Fitness: 0.00|Params: [2.0616037e-05 4.2061394e-05]\n", + "Full_iAMaLGaM - # Gen: 25|Fitness: 0.00|Params: [1.1449511e-06 2.9405437e-06]\n", + "Full_iAMaLGaM - # Gen: 30|Fitness: 0.00|Params: [-9.1895004e-07 -1.6921636e-06]\n", "====================\n", - "Indep_iAMaLGaM - # Gen: 5|Fitness: 0.17|Params: [0.61008203 0.38497373]\n", - "Indep_iAMaLGaM - # Gen: 10|Fitness: 0.14|Params: [0.69136626 0.5000103 ]\n", - "Indep_iAMaLGaM - # Gen: 15|Fitness: 0.14|Params: [0.69136626 0.5000103 ]\n", - "Indep_iAMaLGaM - # Gen: 20|Fitness: 0.14|Params: [0.69136626 0.5000103 ]\n", - "Indep_iAMaLGaM - # Gen: 25|Fitness: 0.14|Params: [0.63670754 0.41385096]\n", - "Indep_iAMaLGaM - # Gen: 30|Fitness: 0.14|Params: [0.63670754 0.41385096]\n", + "Indep_iAMaLGaM - # Gen: 5|Fitness: 0.05|Params: [0.14939058 0.30428362]\n", + "Indep_iAMaLGaM - # Gen: 10|Fitness: 0.02|Params: [0.13289806 0.28522342]\n", + "Indep_iAMaLGaM - # Gen: 15|Fitness: 0.02|Params: [0.13289806 0.28522342]\n", + "Indep_iAMaLGaM - # Gen: 20|Fitness: 0.02|Params: [0.13289806 0.28522342]\n", + "Indep_iAMaLGaM - # Gen: 25|Fitness: 0.02|Params: [0.13289806 0.28522342]\n", + "Indep_iAMaLGaM - # Gen: 30|Fitness: 0.02|Params: [0.13289806 0.28522342]\n", "====================\n", - "MA_ES - # Gen: 5|Fitness: 840.14|Params: [ 1.6348459 -0.22510758]\n", - "MA_ES - # Gen: 10|Fitness: 839.89|Params: [ 1.6386213 -0.21230145]\n", - "MA_ES - # Gen: 15|Fitness: 839.11|Params: [ 1.6380427 -0.21285582]\n", - "MA_ES - # Gen: 20|Fitness: 839.04|Params: [ 1.6380086 -0.21284352]\n", - "MA_ES - # Gen: 25|Fitness: 839.04|Params: [ 1.6380068 -0.21284315]\n", - "MA_ES - # Gen: 30|Fitness: 839.04|Params: [ 1.6380068 -0.21284278]\n", + "MA_ES - # Gen: 5|Fitness: 0.33|Params: [-0.5359075 -0.7642154]\n", + "MA_ES - # Gen: 10|Fitness: 0.33|Params: [-0.5359075 -0.7642154]\n", + "MA_ES - # Gen: 15|Fitness: 0.33|Params: [-0.5359075 -0.7642154]\n", + "MA_ES - # Gen: 20|Fitness: 0.33|Params: [-0.5359075 -0.7642154]\n", + "MA_ES - # Gen: 25|Fitness: 0.33|Params: [-0.5359075 -0.7642154]\n", + "MA_ES - # Gen: 30|Fitness: 0.33|Params: [-0.5359075 -0.7642154]\n", "====================\n", - "LM_MA_ES - # Gen: 5|Fitness: 6.08|Params: [-1.30967 1.6290693]\n", - "LM_MA_ES - # Gen: 10|Fitness: 6.08|Params: [-1.30967 1.6290693]\n", - "LM_MA_ES - # Gen: 15|Fitness: 6.08|Params: [-1.30967 1.6290693]\n", - "LM_MA_ES - # Gen: 20|Fitness: 6.08|Params: [-1.30967 1.6290693]\n", - "LM_MA_ES - # Gen: 25|Fitness: 6.08|Params: [-1.30967 1.6290693]\n", - "LM_MA_ES - # Gen: 30|Fitness: 6.08|Params: [-1.30967 1.6290693]\n", + "LM_MA_ES - # Gen: 5|Fitness: 7.78|Params: [-2.7078676 1.8501627]\n", + "LM_MA_ES - # Gen: 10|Fitness: 7.45|Params: [-2.7272193 1.9920657]\n", + "LM_MA_ES - # Gen: 15|Fitness: 7.42|Params: [-2.7236936 1.9769124]\n", + "LM_MA_ES - # Gen: 20|Fitness: 7.42|Params: [-2.7237751 1.9702088]\n", + "LM_MA_ES - # Gen: 25|Fitness: 7.41|Params: [-2.721651 1.9702324]\n", + "LM_MA_ES - # Gen: 30|Fitness: 7.41|Params: [-2.7209196 1.9683607]\n", "====================\n", - "RmES - # Gen: 5|Fitness: 1.81|Params: [ 0.17560971 -0.0752801 ]\n", - "RmES - # Gen: 10|Fitness: 0.12|Params: [0.7972345 0.60836744]\n", - "RmES - # Gen: 15|Fitness: 0.09|Params: [0.9898771 1.0095383]\n", - "RmES - # Gen: 20|Fitness: 0.01|Params: [0.9717485 0.9363907]\n", - "RmES - # Gen: 25|Fitness: 0.01|Params: [0.94771284 0.9043704 ]\n", - "RmES - # Gen: 30|Fitness: 0.00|Params: [1.0041738 1.006372 ]\n", + "RmES - # Gen: 5|Fitness: 0.37|Params: [-0.59044695 -0.84810144]\n", + "RmES - # Gen: 10|Fitness: 0.37|Params: [-0.59044695 -0.84810144]\n", + "RmES - # Gen: 15|Fitness: 0.11|Params: [-0.33058548 -0.5489675 ]\n", + "RmES - # Gen: 20|Fitness: 0.11|Params: [-0.33058548 -0.5489675 ]\n", + "RmES - # Gen: 25|Fitness: 0.11|Params: [-0.33058548 -0.5489675 ]\n", + "RmES - # Gen: 30|Fitness: 0.09|Params: [-0.28109062 -0.47365618]\n", "====================\n", - "GLD - # Gen: 5|Fitness: 2.20|Params: [-0.1850586 0.12321383]\n", - "GLD - # Gen: 10|Fitness: 1.01|Params: [-0.00243703 0.00842408]\n", - "GLD - # Gen: 15|Fitness: 0.40|Params: [0.364061 0.1313454]\n", - "GLD - # Gen: 20|Fitness: 0.27|Params: [0.5125 0.2448907]\n", - "GLD - # Gen: 25|Fitness: 0.16|Params: [0.6050571 0.37277922]\n", - "GLD - # Gen: 30|Fitness: 0.10|Params: [0.68510354 0.4659995 ]\n", + "GLD - # Gen: 5|Fitness: 0.01|Params: [-0.1030587 -0.19583313]\n", + "GLD - # Gen: 10|Fitness: 0.01|Params: [-0.07785733 -0.14456296]\n", + "GLD - # Gen: 15|Fitness: 0.00|Params: [-0.03990307 -0.08063483]\n", + "GLD - # Gen: 20|Fitness: 0.00|Params: [-0.0303201 -0.0579391]\n", + "GLD - # Gen: 25|Fitness: 0.00|Params: [-0.03103314 -0.06153299]\n", + "GLD - # Gen: 30|Fitness: 0.00|Params: [-0.0139228 -0.02847508]\n", "====================\n", - "SimAnneal - # Gen: 5|Fitness: 19.24|Params: [-1.131186 0.8961963]\n", - "SimAnneal - # Gen: 10|Fitness: 3.70|Params: [-0.8958996 0.83507967]\n", - "SimAnneal - # Gen: 15|Fitness: 3.08|Params: [-0.7558189 0.5737612]\n", - "SimAnneal - # Gen: 20|Fitness: 2.34|Params: [-0.52904546 0.27448243]\n", - "SimAnneal - # Gen: 25|Fitness: 1.89|Params: [-0.36856776 0.14812674]\n", - "SimAnneal - # Gen: 30|Fitness: 1.23|Params: [-0.07835681 0.03237222]\n", + "SimAnneal - # Gen: 5|Fitness: 114.55|Params: [-1.7434927 0.60876817]\n", + "SimAnneal - # Gen: 10|Fitness: 29.11|Params: [-1.990768 0.48308632]\n", + "SimAnneal - # Gen: 15|Fitness: 4.59|Params: [-2.1422086 0.30636734]\n", + "SimAnneal - # Gen: 20|Fitness: 4.36|Params: [-2.0813289 0.18672767]\n", + "SimAnneal - # Gen: 25|Fitness: 4.25|Params: [-2.0612166 0.12882923]\n", + "SimAnneal - # Gen: 30|Fitness: 3.98|Params: [-1.9781697 -0.0172411]\n", + "====================\n", + "GESMR_GA - # Gen: 5|Fitness: 9.00|Params: [-1.181883 -1.2426325]\n", + "GESMR_GA - # Gen: 10|Fitness: 1.43|Params: [-1.1773776 -0.99034745]\n", + "GESMR_GA - # Gen: 15|Fitness: 0.43|Params: [-0.57461786 -0.78734714]\n", + "GESMR_GA - # Gen: 20|Fitness: 0.27|Params: [-0.5132652 -0.7691257]\n", + "GESMR_GA - # Gen: 25|Fitness: 0.24|Params: [-0.4847008 -0.7374786]\n", + "GESMR_GA - # Gen: 30|Fitness: 0.21|Params: [-0.46073768 -0.7048114 ]\n", + "====================\n", + "SAMR_GA - # Gen: 5|Fitness: 0.59|Params: [0.68800676 1.8831477 ]\n", + "SAMR_GA - # Gen: 10|Fitness: 0.57|Params: [0.64244306 1.7375212 ]\n", + "SAMR_GA - # Gen: 15|Fitness: 0.57|Params: [0.64244306 1.7375212 ]\n", + "SAMR_GA - # Gen: 20|Fitness: 0.57|Params: [0.64244306 1.7375212 ]\n", + "SAMR_GA - # Gen: 25|Fitness: 0.52|Params: [0.6906921 1.8796452]\n", + "SAMR_GA - # Gen: 30|Fitness: 0.52|Params: [0.6906921 1.8796452]\n", "====================\n" ] } @@ -191,7 +199,7 @@ "\n", "for s_name in [\"SimpleES\", \"SimpleGA\", \"PSO\", \"DE\", \"Sep_CMA_ES\",\n", " \"Full_iAMaLGaM\", \"Indep_iAMaLGaM\", \"MA_ES\", \"LM_MA_ES\",\n", - " \"RmES\", \"GLD\", \"SimAnneal\"]:\n", + " \"RmES\", \"GLD\", \"SimAnneal\", \"GESMR_GA\", \"SAMR_GA\"]:\n", " strategy = Strategies[s_name](popsize=20, num_dims=2)\n", " es_params = strategy.default_params\n", " es_params = es_params.replace(init_min=-2, init_max=2)\n", @@ -213,69 +221,28 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# XNES on Sinusoidal Task" + "# Try out one of the many `evosax` algorithms!" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 5, "metadata": {}, "outputs": [ { - "name": "stdout", - "output_type": "stream", - "text": [ - "xNES - # Gen: 500|Fitness: -0.00000|Params: [ 9991.45 -9987.809]\n", - "xNES - # Gen: 1000|Fitness: -0.00000|Params: [ 9951.659 -9911.333]\n", - "xNES - # Gen: 1500|Fitness: -1.00000|Params: [ 8.5644424e-05 -5.0786706e-03]\n", - "xNES - # Gen: 2000|Fitness: -1.00000|Params: [ 8.5644424e-05 -5.0786706e-03]\n", - "xNES - # Gen: 2500|Fitness: -1.00000|Params: [ 8.5644424e-05 -5.0786706e-03]\n", - "xNES - # Gen: 3000|Fitness: -1.00000|Params: [ 8.5644424e-05 -5.0786706e-03]\n", - "xNES - # Gen: 3500|Fitness: -1.00000|Params: [ 8.5644424e-05 -5.0786706e-03]\n", - "xNES - # Gen: 4000|Fitness: -1.00000|Params: [ 8.5644424e-05 -5.0786706e-03]\n", - "xNES - # Gen: 4500|Fitness: -1.00000|Params: [ 8.5644424e-05 -5.0786706e-03]\n", - "xNES - # Gen: 5000|Fitness: -1.00000|Params: [ 8.5644424e-05 -5.0786706e-03]\n" - ] + "data": { + "text/plain": [ + "dict_keys(['SimpleGA', 'SimpleES', 'CMA_ES', 'DE', 'PSO', 'OpenES', 'PGPE', 'PBT', 'PersistentES', 'ARS', 'Sep_CMA_ES', 'BIPOP_CMA_ES', 'IPOP_CMA_ES', 'Full_iAMaLGaM', 'Indep_iAMaLGaM', 'MA_ES', 'LM_MA_ES', 'RmES', 'GLD', 'SimAnneal', 'SNES', 'xNES', 'ESMC', 'DES', 'SAMR_GA', 'GESMR_GA', 'GuidedES', 'ASEBO', 'CR_FM_NES', 'MR15_GA'])" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ - "from evosax.strategies import XNES\n", - "\n", - "def f(x):\n", - " \"\"\"Taken from https://github.com/chanshing/xnes\"\"\" \n", - " r = jnp.sum(x ** 2)\n", - " return -jnp.sin(r) / r\n", - "\n", - "batch_func = jax.vmap(f, in_axes=0)\n", - "\n", - "rng = jax.random.PRNGKey(0)\n", - "strategy = XNES(popsize=50, num_dims=2)\n", - "es_params = strategy.default_params\n", - "es_params = es_params.replace(use_adaptive_sampling=True, \n", - " use_fitness_shaping=True,\n", - " eta_bmat=0.01,\n", - " eta_sigma_init=0.1)\n", - "\n", - "state = strategy.initialize(rng, es_params)\n", - "# Set mean to a bad initial guess\n", - "state = state.replace(mean = jnp.array([9999.0, -9999.0]))\n", - "num_iters = 5000\n", - "for t in range(num_iters):\n", - " rng, rng_iter = jax.random.split(rng)\n", - " y, state = strategy.ask(rng_iter, state, es_params)\n", - " fitness = batch_func(y)\n", - " state = strategy.tell(y, fitness, state, es_params)\n", - " if (t + 1) % 500 == 0:\n", - " print(\"xNES - # Gen: {}|Fitness: {:.5f}|Params: {}\".format(\n", - " t+1, state.best_fitness, state.best_member))\n" + "Strategies.keys()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/examples/02_mlp_control.ipynb b/examples/02_mlp_control.ipynb index 061912e..f689e92 100755 --- a/examples/02_mlp_control.ipynb +++ b/examples/02_mlp_control.ipynb @@ -35,14 +35,6 @@ "execution_count": 1, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - ":228: RuntimeWarning: scipy._lib.messagestream.MessageStream size changed, may indicate binary incompatibility. Expected 56 from C header, got 64 from PyObject\n", - "WARNING:absl:No GPU/TPU found, falling back to CPU. (Set TF_CPP_MIN_LOG_LEVEL=0 and rerun for more info.)\n" - ] - }, { "name": "stdout", "output_type": "stream", @@ -57,7 +49,7 @@ "\n", "from evosax import OpenES, ParameterReshaper, FitnessShaper, NetworkMapper\n", "from evosax.utils import ESLog\n", - "from evosax.problems import GymFitness\n", + "from evosax.problems import GymnaxFitness\n", "\n", "rng = jax.random.PRNGKey(0)\n", "network = NetworkMapper[\"MLP\"](\n", @@ -83,8 +75,8 @@ "metadata": {}, "outputs": [], "source": [ - "evaluator = GymFitness(\"CartPole-v1\", num_env_steps=200, num_rollouts=16)\n", - "evaluator.set_apply_fn(param_reshaper.vmap_dict, network.apply)" + "evaluator = GymnaxFitness(\"CartPole-v1\", num_env_steps=200, num_rollouts=16)\n", + "evaluator.set_apply_fn(network.apply)" ] }, { @@ -95,7 +87,7 @@ { "data": { "text/plain": [ - "EvoParams(opt_params=OptParams(lrate_init=0.01, lrate_decay=0.999, lrate_limit=0.001, momentum=0.9, beta_1=None, beta_2=None, eps=None, max_speed=None), sigma_init=0.04, sigma_decay=0.999, sigma_limit=0.01, init_min=0.0, init_max=0.0, clip_min=-3.4028235e+38, clip_max=3.4028235e+38)" + "EvoParams(opt_params=OptParams(lrate_init=0.05, lrate_decay=1.0, lrate_limit=0.001, momentum=0.0, beta_1=None, beta_2=None, beta_3=None, eps=None, max_speed=None), sigma_init=0.03, sigma_decay=1.0, sigma_limit=0.01, init_min=0.0, init_max=0.0, clip_min=-3.4028235e+38, clip_max=3.4028235e+38)" ] }, "execution_count": 3, @@ -119,29 +111,26 @@ "name": "stderr", "output_type": "stream", "text": [ - "/Users/rob/anaconda3/envs/mle-toolbox/lib/python3.9/site-packages/jax/_src/tree_util.py:188: FutureWarning: jax.tree_util.tree_multimap() is deprecated. Please use jax.tree_util.tree_map() instead as a drop-in replacement.\n", - " warnings.warn('jax.tree_util.tree_multimap() is deprecated. Please use jax.tree_util.tree_map() '\n" + "/Users/rob/anaconda3/envs/mle-toolbox/lib/python3.9/site-packages/flax/core/scope.py:740: FutureWarning: jax.tree_leaves is deprecated, and will be removed in a future release. Use jax.tree_util.tree_leaves instead.\n", + " abs_value_flat = jax.tree_leaves(abs_value)\n", + "/Users/rob/anaconda3/envs/mle-toolbox/lib/python3.9/site-packages/flax/core/scope.py:741: FutureWarning: jax.tree_leaves is deprecated, and will be removed in a future release. Use jax.tree_util.tree_leaves instead.\n", + " value_flat = jax.tree_leaves(value)\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "Generation: 0 Generation: 21.875\n", - "Generation: 20 Generation: 80.25\n", - "Generation: 40 Generation: 82.75\n", - "Generation: 60 Generation: 166.1875\n", - "Generation: 80 Generation: 200.0\n", - "Generation: 100 Generation: 200.0\n", - "Generation: 120 Generation: 200.0\n", - "Generation: 140 Generation: 200.0\n", - "Generation: 160 Generation: 200.0\n", - "Generation: 180 Generation: 200.0\n" + "Generation: 0 Generation: 22.875\n", + "Generation: 20 Generation: 81.75\n", + "Generation: 40 Generation: 200.0\n", + "Generation: 60 Generation: 200.0\n", + "Generation: 80 Generation: 200.0\n" ] } ], "source": [ - "num_generations = 200\n", + "num_generations = 100\n", "print_every_k_gens = 20\n", "\n", "es_logging = ESLog(param_reshaper.total_params,\n", @@ -151,7 +140,7 @@ "log = es_logging.initialize()\n", "\n", "fit_shaper = FitnessShaper(centered_rank=True,\n", - " z_score=True,\n", + " z_score=False,\n", " w_decay=0.1,\n", " maximize=True)\n", "\n", @@ -188,7 +177,7 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAADgCAYAAADsbXoVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABaeElEQVR4nO2deZxcVZX4v+fVXr13ujvp7HtCQkgISdhEwg4uEAGVRSGMuwgyDo6M8lNEdMCFQWccRhFEEAmCGlBAECRA2EICIRvZyJ70vta+3t8f93V19V6d9JLlfj+f+lTVffe+d97r6nfeOffcc0QphcFgMBgMANZwC2AwGAyGwwejFAwGg8GQwSgFg8FgMGQwSsFgMBgMGYxSMBgMBkMGoxQMBoPBkMEoBcNRiYgsFpF9wy2HwXCkYZSC4ZAQkatEZLWIBEWkSkSeFZEPHcL+lIhMzfq+WETS9v4DIrJFRK4bGOl7lOFBW45LOrX/l92+1P6+VERW9rCPFSISteWuF5E/i0hlL8f8mIisEpGQiDSIyCMiMnZAT6wP7PNOdpZTRG4Tkd/3MS5un2vb672s7Z8Tkc32369GRJ4RkYLBPBfDwWOUguGgEZFvAPcAPwJGAuOB/wUu6WVYT/ty9rL5gFIqHygEvgXcJyKz+i1w/9gKXNNJvk8BH/RjH1+z5Z4OFAP/1V0nEbkc+AP6WpYBs4EYsFJESg5C9n4jInnAZUAL8JmD2MWPlVL5Wa+59n7PRP8+rlRKFQDHAY8NlNyGgccoBcNBISJFwO3A9UqpPyulQkqphFLqr0qpb9p9FonIGyLSbFsR/yMi7qx9KBG5XkS2AdtE5BV703v20+ans4+pNMuBJmCWiHhE5B4ROWC/7hERTw/yjhaRP4lInYjsFJEb+zjFvwIfyropXwisA6r7d6VAKdUI/Ak4vhu5BPgZcIdS6g9KqYhSqhr4PBAE/tXut1REXrOvYYv95H1O1n6KROR++zrvF5E7RMSRNXaliPxURJrs87+okyiXAc3ov+m1/T3HXlgIvKGUerftWiilfqeUCgzgMQwDiFEKhoPlVMAL/KWXPin0Ta3M7n8O8NVOfZYAJwOzlFIfttvm2k+bHZ4oRcQSkU+gn7rXA98BTgHmAXOBRcCtnYUQEQt9k38PGGPLcZOIXNCL7FHgSeAK+/s1wEO99O8RESlD33Tf7WbzDLSF9Xh2o1IqjVYk52U1n4y2VMqA7wF/FpFSe9uDQBKYCpwInI9WLNljt9hjfwzcbyukNq4FHgWWATNF5KT+nmcPvAVcICLfF5HTe1LahsMHoxQMB8sIoF4pleypg1JqjVLqTaVUUim1C/gVcGanbv9pPz1GejnWaBFpBurRN8PPKqW2AFcDtyulapVSdcD3gc92M34hUK6Uul0pFVdK7QDuo/2G3xMPAdeISLEt9/I++nfmF7bc7wFVwDe66VNmv1d1s60qaztALXCPbZE9hr7Jf1RERgIfAW6yLbZatKsq+/x2K6XuU0qlgN8BlWiXHyIyHjgL+INSqgZ4kSzXWY7cbFuEba/fASilXgUuBeYDTwMNInJ3mxVjOPzozY9rMPRGA1AmIs6eFIOITAfuBhYAfvTvbU2nbntzONYBpVR3k66jgd1Z33fbbZ2ZQLtiacMBvNrbQZVSK0WkHG2R/E0pFen4cN0nNyqlftNHn3r7vRLY2WlbZdZ2gP2qYwbLtvOdALiAqiz5LDpe24zbSykVtvvl202fBd5XSq21vz8C/ExEblZKJbIFEpFvA9+2v/5eKfVl+/NPlVJdrDT7eM8Cz9oW21loq2gL+iHBcJhhLAXDwfIGejJ0SS997gU2A9OUUoXom0nnu+qhpOk9gL4htjHebuvMXmCnUqo461WglPpIDsf4PfBvHKTrKAe2APuAT2Y32jfQy9BP7W2M6eTyaTvfvei/RVnW+RUqpWbnKMM1wGQRqRaRarQiL0NbHx1QSv0oazL5y52394ZSKq2UehH4J93MrxgOD4xSMBwUSqkW4LvAL0VkiYj4RcQlIheJyI/tbgVAKxAUkZnAV3LYdQ0wOUcxHgVuFZFy22//XfRNvDOrgICIfEtEfCLiEJHjRWRhDsf4Bdqv/0oP20VEvNmvHGUH9OQ5cLN9HlfZ+xgF/AYdbZUdsVQB3Ghf50+iI3meUUpVAc+jn+4L7bmXKXbkT6+IyKnAFPR8zDz7dTw6Gqq/LqTu9n+JiFwhIiWiWYR2xb15qPs2DA5GKRgOGqXUz9B+8luBOvQT69do973fDFwFBNA+/FxCEW8Dfmf7pT/VR987gNXoqKD1wDt2W2c5U8DH0De8nWiXzG+Aor6Esec7XuzktsnmNCCS/ZLew2u7O8ZjaBfOv6LdcpsAH3C6Uqohq+tbwDRb/h8Cl2dtvwZw22ObgCfQ7qe+uBZ4Uim1XilV3fYCfg58LGsiuy/+XTquU2hzezUBXwC2oR8Qfg/8RCn1SI77NQwxYorsGAyHP6IXzH1eKXXQCwMNhlwwloLBYDAYMhilYDAYDIYMxn1kMBgMhgzGUjAYDAZDBqMUDAaDwZDhiF7RfOGFF6q///3vwy2GwWAwHGn0uDT/iLYU6uvr++5kMBgMhpw5opWCwWAwGAYWoxQMBoPBkGHQlIKIjBORl0Rkk4hsFJGv2+2lIvIPEdlmv5fY7SIivxCR7SKyTkTmD5ZsBoPBYOiewbQUksC/KaVmoQuhXC+6hOItwItKqWnoDJC32P0vQud1mQZ8EZ1h02AwGAxDyKBFH9mZG6vszwEReR9d9eoSYLHd7XfACnTd3UuAh+zEY2+KSLGIVNr7MRzFfOu5B2huLqPQmtB3526IJvZyyt9/gCcU69DeWuLjpbOvoMx5Cs5+JC9NpsLkb76DCbsbKGqNIyg6Z/huLPOz8oxLGOFenNl3XAXYH3+Zs1f+nYq69mqTe8Z52XR8JXkFX8NtldEXSqXZnvgj8za+zfFbTDCFoXtk7glc9t2Bz+g+JCGpIjIRXSLwLWBk1o2+Grv6E1phZBcF2We3dVAKIvJFtCXB+PHjB09ow5Dx9IH/heB8CoOf7rtzJxK+VRxf9wjzV6VoyYOUbfu6UlCwMcaTU+9jR94HqOAlOe9zXvD/+OKfdpAWaMnvWvDBUjB7Y4y6xO94bd5WmkNXARDL/yufWv08Z72lMrI4U7rvlPUt3HPZNwiFv4wzMb3X44tjG1eueZJz3lMEfRA/ogPHDYPF3rI9g7LfQf+5iUg+utbsTUqp1uwaIUopJSL9yrOhlPo18GuABQsWmBwdRzjhRAyxYkwf5eVPnzyn7wGduP6FP3P23WksP5z8+lrEo0sApwIBtn3odD7yToKmT+/llsty23ckGeGub+iqmdOWL8M9/QToVG1NKUXVzddz6dMvkVe5lpt+eD8At33vNi5+S1Fy2lhm/uY5xLJQ6TQtf/kL3Horn3k5xqMff5BnrnwVt8Pdowx/WPYkJz6g8C4+jpk/fzRzTgbDUDCo0Uci4kIrhEeUUn+2m2tEpNLeXomuOwuwHxiXNXys3WY4itnf0giAw5E6qPHhne9z3I40pRec1uHm6SgooOjiSzj5fUg2N+W8vz9v+zOjDiRI5wnuGXO7KAQAEaHyrl8Q9wi+HWGo3059pJ6SrQ0kPTDyV08jlv7XEsui+LLLKP3sNZy6TijcE+KpN3/cZZ/Z1Kx/B4DKr37bKATDkDOY0UcC3I+u/Xp31qan0IU9sN+fzGq/xo5COgVoMfMJRz/7W3WNGMvqv1JQSjHtrRoQKP5K1/LAxZ++Ak8CKte15LzPF3Y9z3FVaQonjeq1nzidxEeNoKBFaF31K97Y9QLjaxWOMcWIq6sVUHbjjbjKK/jCS2l+s+VREoHqbvYKiVSC9L4mUg7wHDc3Z7kNhoFiMC2F09HVpM4WkbX26yPAncB5IrINONf+DvAMsAPYjq7S9dVBlM1wmFAVsAuHSbLfY5tjzYyuVURHOHGNn9Rlu+/42cRd4G+I57zPQP1eyhvAO6fvG7Jr4gRGNSm2vv84r63/PRPqoOT4k7rt68jPo/iKTzNxryIaFK7826fZ3rS9S78NdesYU6NIjfQhLlfOchsMA8WgKQWl1EqllCilTlBKzbNfzyilGpRS5yilpimlzlVKNdr9lVLqeqXUFKXUHKXU6sGSzXD4UBOyXTsHoRQOBA9QFFJYhT1HFiVcAvHcrZCCvY0IgvfU8/rsWzJ1FhXNsM7pZvP+D/DFwTu/58JoRR//OAB3bfRRF23k2yu/3aXPqh3PMrFWUTQ11zLVBsPAYlY0G4aV+vAhKIXAXopD4Cku7LFP0m1hJRTEgn3uL5lOUrE/AYDvxIV99i+achwOBcsln+IG/a/kmTGjx/7ucePwnXgi5Tv8XN3UzPuN79MYbezQZ9e2NykKQ+H80/s8vsEwGBilYBhWGiPNAKQ5CKXQsJniEORX9Oz/T3mcWEmBYE2H9r2BvV1uyM2xZiZXK+IFDpzl5X0e3z1Rr6soqo+xOKLDoz3Teg83Lbr448T21nJKrZ5AXrXtrx22p3ccAMB7kinFbBgejFIwDCst8Rauez7F9HX9X6RVXbMFbwL8o3per5JyO3EkgE4Tuzf+80buWXMPAMu3L+eF3S/QFG1iZLMiXeHL6fhue53MmCaLU2PjcI0diyM/r9cxhR/7GI6iIop2HUd+Os1bL98Ote9rWdMpfFV6AZ5n5sycZDAYBhqjFAzDSiDWwnnvKqZu6L9SaKnSax0doyf22CftdeNKCgQ6BrLVR+qpDeto6N9u+C0Pb3qYxnA9BRFwFPR+Y2/DMWIEkpfHdUUX4tx5AM/03q0E0KGyI77wecJrt/OxyCze9Lpg2/MAVAUPMKZeES924SgoyEkGg2GgMUrBMKyoQAPONFiJ/ruPQg1akTjHTe15/z4fziQdLAWlFMF4gBZbKTTHmjkQOkBTYB/5EfAUFeV0fBHBM2EC1srVxD/4AN8JJ+Q0ruTqq3GUl3HuS0H2uVzs3/sGALtr3qWyUWGNHpHTfgyGwcAoBcOw4m9bp5Ds3+J0pRTJlhAAzlFjeu7o8+JKgGo9kGmKp+MkVYrmlj0opWiNtVIbrqW2aRf5UfCW9J2fqA33xAkkq6txT55M6TWfzWmM5fMxYulSCtfvYkKNYm3DRgB2Va2hshHyJ03J+fgGw0BjlIJhWMkPtALg6KdSaI234gml9dgRPT9Zi8+HJwHJYLulEIjrZHUtKkEoESKpkqRVmh1VG7EU5JVV5iyHZ/p0xOVizE9/guX35zyu+JOfRHw+PvZ2mm3JFgjVU71nE/44FM40WeMNw4dRCoZhJS8cBvqvFGrDtRSHFApwlpb22E98XtwJCAfb5xRCMa2IAqQ7RCDtrdkJgLM8d6VQet11THnu73hnzeqX/I7CQoovvZTTNyn2J9xw4F2Ce/cB4JlxfL/2ZTAMJEYpGIaV/JBebexMAqnc5xVC8SDFIVB5LsTZc15Hy+vDm4BIVkhqMFTDzL2KygbF3kB7Yt7GZu3KclT04o7qvH+PB9fo0Tn3z6ZoyRKcKZAaF+x/h1StXkvhnjjxoPZnMAwERikYho1oIk5hRK82dqUgGaztY0Q7wWAVRSGgqHeXjeX34U5CJFiXNbaar/4txeWvpdndujvTnhfR1oqjfGw/zuLgcU/QIa2OkEXDB8/ja0qRdnDQSsZgGAiMUjAMGwcCTRTbC43dSYgHDvQ+IItg6z6KgwpHSc+rmQEcfh1eGo1FIKrdRsFwHflR8MdgT8uuTN/8qD2mdGiif6yCAtJ+L2WtiicCWxnVBKkReb1aPgbDYGOUgmHY2N/SoJ/2AVcS4sHuM4d2RyhcR1EYXGW9Rwq5bKUQSTmgebc9th5/DDxx2N2yA4Ax+WMoiOgxjuLcQlIPFRHBOWoUZa1w34hyxjZC0fQ5Q3Jsg6EnjFIwDBtVwUaKQ9pl40pCPNQP91G4nqIQuCt6nxR2+vUisHjagqZdAIRa6rEU+OKKPa26etWMkhnktymFwt6tj4HEN2YcFQGLeCpOZYsD71SzktkwvBilYBg2akNNFNuWgjsJ8VBN7wOyiLY04E2At7L3kqzuPK0UYinJKIVYs4448sbhQKSWfFc+4wrGURBRJD0ypO4b1+jRlAeEipATRzyZyadkMAwXRikYho2GQD2FOiIVVwriobreB2SRaNQ3dteocb3287QpBfFnlEI8oIvueBOQVCmKPEVU5leSH9XRTEOJq3IUecEk3/ZequWd3nOWVYNhKDBKwTBsRKp3YylI+pzafRRuyHlsqlUvQHMUl/Taz52nXUFJZxE06nUI8aAe67Nr7xR7ihnnryQ/AlZez7UZBgNXpXZ/Td/QBA4H3uOM+8gwvBilYBg2Anv0xK8qy8eZhngod6WQDOsJAEdh74njvPl60jhh5WUshWRIj/XGFShFkaeI04pncFwgQUEfSmagcdpKIfjyK3imTsXy5Zah1WAYLIxSMAwLDcEYUm9PLI/UIaDxYFPO45W96M3qI5uoL68YgBQ+aN4D6RQqotNTW0pwJaHIU4Qz0khBBJzFxf07kUOkbU2CCofxzjErmQ3Dj1EKhmHh7xurKYnoWWap1EVyEqGWnMerqF793FeKaXe+3p5KuyCdgNYDqGgis92b0O4jGj4gFbdwlOaeDG8gcFVUgAgAvuONUjAMP2aVjGHAWL1vO69s30+5p+8ImkdX7eWCpP3EPkavIE6EO5bMXL59OXmuPM6b0LVeshXVK6H7shTEdsek0w79CNS0EyLtNZu9cW0pqHd+TzouOEb3nIZ7MBC3G2d5OcnaWrzHmzUKhuHHKAXDgBBPpvnS098nouoJ7/paTmMuw04rUTYSBSQTcV1L2ZMPwEObHsJluboqhVQCKw5KwMrrvSBOm49eJUX/2pt2YcXSme2+OBSnFKlNLwKjcJQMfS0DZ+UoUk1NeKdPG/JjGwydMUrBMCD8/s3dhJNBRo2Ax289t8/+lgjP3fh9Ek7IyysgDiRTAuH6jFKIJqPsDu0mkU7gsrJCRcMNOGOQ9DgQq3cPqHh1NJFKpMHngMadOOLtGVm9cSg6sJZUTO9nqFYzZ+NfsABneTnidg/5sQ2GzhilYOjAl5+8h1hwMvlW7/H/nXl5Sy1FExVuZ5qyfE9ug+JJEk5wef1aKaQtiDSDHQAUTUaJp+PsatnFtJL2p2gVrMMVF5L+vtcUiAhxl0A0BlNmoPa/gysmYFsp3riieNcbpCtOBnbiGOKJZoCR3/zmkB/TYOgJoxQMHVjZ9Fus1jMpi1/ar3GTyvNRhRaBZDznMRJPkXSB06sznaZSApH2CKRoUmeo29y4uYNSiAQP4I9B2p+b8kl4HEg0DhNOI/rO7/DFK1ACovREc1Ggllj+xcBOnCNH5Sy/wXA0YpSCIUM8mUQkzewxfh69bHG/x3/8L3cST+WuFKxEmpRTcPlspZDuqBQiSb2eYEvjFj4+5eOZ9mDrAfwxBfm9zye0kXQ7kJhWCsF3HsAfg2SBB1drjDNboszwCXW7ozhKS/FMG9qJZoPhcMOEpBoyRBI6VFNJqo+ePYxPRkikE313tHHE06Rcgsen5xCyLYVEOkFS6bDTLU1bOowLBqvJi4L0EXnURsrjxIolYPxpBC0LX0yRLtEK5WPNYVxTziH01mryTjmlzzkKg+Fox/wHGDKEE/opX5H7jT2baCraL0vBkVCk3RYuWymk0wLRZr0v23UkSlsKSrVPDofCtfhjYBUV53SclMeFM5aEwkpCxePwx4AyPTadtIgVnEaqvp6800/LWXaD4WjFKAVDhkhGKeReFjObaDJKSqVIpXOzNJwJhXJbOOwIobRyZiyFNqUwJZGgKdZEbbg9rXYw0oA/Bs6C3CKF0l43jpiWKTByllYoI0aAZZEetYhQjZ6byDvNKAWDwSgFQ4Zo8uCVQlqliaX0YrRcXUjOJCiXA/Hom7JSroxSiCR0+tRZMb3P7FrKoTalUJSbUlAeF664Vgqh0fPwx8BVWoHl96NGLSK0ajXuSZMyyekMhmOZQVMKIvKAiNSKyIastttEZL+IrLVfH8na9h8isl1EtojIBYMll6Fn2iyF9EG4j9qe7AHi6b5dSEopXAnA7crE5yvl1CGpQCSs02iXpfTNPJwMZ8aGAq1YCjzFpbkJ5/XgiusFa4HiibhS4C4px/L7SYVCxLZvN3mHDAabwbQUHgQu7Kb9v5RS8+zXMwAiMgu4Aphtj/lfEXEMomyGbmhXCv23FNoihYCc5hViySjeBIjHhWVbCqQdGaUQDdVS3qwY2arnErKVQjSk02F4i3JUCj4v7oQikUqwZe+7ABSVVmL5/aRbAyRranGNGZPbvgyGo5xBUwpKqVeAxhy7XwIsU0rFlFI7ge3AosGSzdA9kTb3UTraR8+uRFPtYxKpvi2NaLQZdxLE6253H6Wt9jmFUD03PJVi2lt6JVskVJ8ZGw/pY/mKc0tJ4fYX4EnAztadbN6zBgBXYRGW30985w5IpTLZSg2GY53hmFP4moiss91LbcnrxwB7s/rss9sMQ0gsaWcejVT1e2x/3UeRSD2eBFheL+JwkHQAWSGpkUg9xSHwBbXCCDft0AOVIhHR+3cVFuck25jyyXgT8PCmh2lqOADoRHqW309sl67pYJSCwaAZaqVwLzAFmAdUAT/r7w5E5IsislpEVtfV5V6+0dA30YSe1E2lY/0fm4ySH1aUBFRO7qNwsB5XChx2wrqk00JStFsKkUa8cXA0x0ApQmHbUkhESNq5ixyFhTnJllcwAncSntz2F3wxPdbKy0fy/GCvzXAb95HBAAyxUlBK1SilUkqpNHAf7S6i/UB2sp2xdlt3+/i1UmqBUmpBeXn54Ap8jBG3I36Sqv8TzZFkhKUvpPnX5amcLIVYi1bojrbVzC5bKSQjkIgSiTbpcpnRBIVRRTjWbB+okXRc/2yt/PycZLP8+hi+GJSl9GdHQT6OrAyrThN5ZDAAQ6wURCT7P+8TQFtk0lPAFSLiEZFJwDRg1VDKZoB4Qk8WJ1T/VzRHU1GKQ1AYynFOoVWX3nT47dXMbZYCQLSZaLgZjz3fPbZZEY616i/hRlIJO6NpjpaCe9JEACbWW8xy61oPVkEBYisLZ3l5+2S3wXCMM2i5j0TkUWAxUCYi+4DvAYtFZB46ReUu4EsASqmNIvJHYBOQBK5X6iDuTIZDIh6PsmhLmpbydN+dOxFNRvEkFO5kbtFH8WAjPsBl11BOuRw4Evaq5UgT8UBzpu+ogBCOBzPb0gldqayvAjtteGfPBuAm/8cpbUyg/DtxVVZmLAgzn2AwtJOzUhARv1Iq3HdPjVLqym6a7++l/w+BH+a6f8PAk0hGuGl5mucPIu4rkozgSaCVQi7uI/um77aVQtrlQJL2c0CkiWQolOk7MiDUt4WkRhpRcYuky8LKsf6Aq6ICZ3k546sSxLZvxzH3BMTpbFcKZj7BYMjQp/tIRE4TkU3AZvv7XBH530GXzDDkJOIRnGlwJoB4qM/+2URTUbxxrRRycR/FQ9od5M7XAWgptwNHss1SaCYVbj9+RcAibEc3pcONOOJCyte/gjTe2bMJr15NbPMW/PNPAsDy6zkF1xhjKRgMbeQyp/BfwAVAA4BS6j3gw4MplGF4SEX107gzKRDqX2RXJJFlKSQiffZvq8fsLiwDIO12YiVtt1WkiUSkPcS1JCBEbOsjEKzGGweV7++XfN7Zs0keqIJ0Gt/8EwGMpWAwdENOE81Kqb2dmoy//ygkFdM3c2cS0sH+KYVoKqrXHSiItxzos38yohWQt7gCAOVy4cyaU1AR29pwuSgOCGGVBKVoCNfgjwI5Rh610TavgGXhmztPfzRzCgZDF3JRCntF5DRAiYhLRG4G3h9kuQzDQDqmb9SuFCSC1f0aG01E8Nr38VRTt9HEHUi1KYVCvSpZuZ04kmkQC8INqJgOPfJMnEhBa5qwAJEmGiL1+GMKZ46RR220KQXvzJk47OI87okTEbcbz/Tp/dqXwXA0k4tS+DJwPXqF8X70wrPrB1EmwzChbLePOwHxYE2/xsbCQSz7QT/Z1LdCSUb1Ajm3344gctuWgrcYmnZCUkcYeaZNJa8lSURpl1ZDtIm8qE5T0R9cIyvwTJtG/llnZdr8809kxprVuEaZEpwGQxu9Rh/ZSel+rpS6eojkMQwjKm7fqJMQD/VPKSQD7Wmukq31vfTUpGN6jqBtRTNuF65kGorGwo6XkbhLN0+dipVWuCICwRoa4wEmxsCbY96jbCY9uRxEOrSJy9Xv/RgMRzO9Wgr2WoEJItK/UA/DEYlK6MldV1KRCDf0a2wy2Jz5nAr2nQdRxbSvSTJKwY0rCemzvgORRsRei+CZomsm5wWFdKCGhmSQvFg/MqRmIZaFdFIKBoOhI7msU9gBvCYiTwGZOEGl1N2DJpVheEjaeYCSEA/3b6I5FWzNfE6HWnvpqVFxPWdg2VXX8GilEJ+yGO/sS7F2vEHa0quNAfwxiAaraI5FcScl5wI7BoOhf+Qyp/AB8De7b0HWy3C0YddT0Eqhf5ZCOmtdQSocyOFYKZIOEKd+LhGPB1cSXb3toz/DqbwkvC4se1LYG4dw4ABBewLayjc/QYNhMOjTUlBKfR9ARPLt78HBFsowTCSzlIKdrTRX0pH2tQnpWEwvfnPn9TwgniaR5c633G7cKTtFhr8cR0JIel2ZpHX+GEQadxBOtuU9MkrBYBgMclnRfLyIvAtsBDaKyBoRmT34ohmGnESW+yja3K+hKtqeblslBZr39NpfEmmSrnb/vjdPh5hWNdr1DaIpUj53JhOqNw7h2o3EE20ZUo1SMBgGg1zcR78GvqGUmqCUmgD8GzrtteEoQ1LaNeNOQiLaAkrlPjjanu9IpQSadvfa3UooUq72n9/oEp29dP2Bd7QMsRRpnxsry1IIheuIG0vBYBhUclEKeUqpl9q+KKVWAL34BQxHKtI20ZyAGCmI9T1hnBkbz8p3lAIivUQgKdVFKRQU6BDT9w+sJZFO4I2lUX4v4nSiPG68cUXjkv/BEe9fhlSDwdA/clEKO0Tk/4nIRPt1KzoiyXCUYdmWgjMNcSUQy336yIq1p9tWKek9oV4ijDMppD3tU1quMWMBaNy0lkgyovMb2eGq4vfhj8O+VJg8OyWSwygFg2FQyEUp/AtQDvwZ+BNQZrcZjjLErtEMkEgLxHNXCpLIqsGQ6mNstEXnV3K3KwX/SfNRDoux25r5oPkDfHGQPK0UrPx8vDHY1boLvz11YfUzzYXBYMiNXKKPmoAbh0AWwzAjqfY8h8l07paCUgpHvH3+QfqyFKItOBOgitvDjyy/H5k9g+N3vc+bB95kQRySdsI6Ky8Pfxzeq3uPk6IKRDLJ7AwGw8CSS/TRP0SkOOt7iYg8N6hSGYYFK9X+tJ9MSc5zCvF0HE8S0k7t75e01atSUOEmXEnA27EEZunpZzKlGt7a9hLeOJlwVGdBAd64YnvzdoqTbl1K0xrSSrIGwzFDLv9ZZUqp5rYvtuVQMWgSGYYNK9tSSFk5u4+iiQieOKQ8DpIOsS2FnsdGwnW4E2B5fR3a8085FUuBa81GnGky4aiOvAL8Ma1wRqlCHP1Mm20wGHInF6WQFpHxbV9EZAK6xrLhKEOyLIVUKnf3USTSgDeh01+n3BZWyoJ4z5Vbw+F6XXvB3zGIzTdvLrhdLNyqf14uey2CIz8fv50LaUTKa+YTDIZBJJfcR98BVorIy4AAZwBfHFSpDMNCtvso3Y+J5miwBk8C8LpIpdJ6wroX91EoopVCMq/jE7/l8ZB36qmc8vpKQOHM1zd/Ky8Pnz3BXBR3GkvBYBhE+rQUlFJ/B+YDjwGPAicppcycwlGIlc5SCimBWA45jIBIqE0peEi5HDiS9KpQQsF63ClwFBR32Vb0kY/gsiOZRpTqMplWfh5eeyLbFzORRwbDYNKjUhCRCSJSBKCUqkdnSD0fuMak0j46sdLtXsF0yspZKQSDNXgSCvH5SbmdOFL0aimEW3SyPVdB10yn+eecg7j1zyu/WGdItfLycCcUXlw4QzEcBcZSMBgGi94shT9ir1wWkXnA48AeYC7wv4MumWHIsVJZSkE5c3YftQSr8Cb0HIByOXAk6FUpRAPNALjzuyoFR34++WeeqT/b0Udt7qI5/qmkg0GsAmMpGAyDRW9KwaeUaqvA/hngAaXUz4DrgEWDLplhyHFkzykoZ84TzS3hOjxxcOUXk3a7+rQUYiFtgbi7sRQAii+/DHG5cFaOBsjkP/runJtJBwJYxlIwGAaN3pRCdomqs4EXAZRS6e67G450si0F0g6I5+Y+ao424EmAu6CUtMeJK6F6tTJidu0FX2FJt9vzzzyT6avfxjVSRz5b9oR0RdgFSuEwloLBMGj0Fn30TxH5I1AFlAD/BBCRSiDeyzjDEYojDSlLv5O2crcUYi3afZRXAG4XzqSCRM8hqfGwrr3g6WaiuQ3L076wrc1SSFZX6e/GUjAYBo3eLIWb0PmOdgEfUkq1pcEchQ5TNRxFpNMKK6WIt4UQ9GOiuTkRwJMAh9+Haqu1nIpnivZ0JmGn2fYXjshp/23V1xJV1QDGUjAYBpEeLQWllAKWddP+7qBKZBgWEuk0jrQi7hF8UdV3UrssWuJhXCkQnw88bl2kRwRvIgTOroFqqWgSsHDneHNvsxRi27YB4Cwvy+2kDAZDvzEJZAwAJFIKKw1Jl5C2QJL9WNEc00/+ls8PbpetFOh+sjmdJh3X6TTabvZ90RZ9FH5XF+DxTJ+e0ziDwdB/jFIwABBPpnGk9PxywmUhKZXbRLNShO3FZpbfh3g9uJIQp4dMqbFWlJ2yIlel0NYvsXsPrjFjTC0Fg2EQ6ZdSsDOknpBj3wdEpFZENmS1ldpZV7fZ7yV2u4jIL0Rku4isE5H5/TsNw6GSSGn3UdoSUi4HkkRbCn2V5Iy1ErPLMFg+H+Lx4E5CUnpwP0WbUUkhLSBeb06yZSsPz4wZOZ6RwWA4GHJJnb1CRApFpBR4B7hPRO7OYd8PAhd2arsFeFEpNQ0d4nqL3X4RMM1+fRG4NzfxDQNFu6UgpFwWjqQClYJEpNdxKlhHIqV/RuLzIV4vrhTEVA+WQrQFSQhxj4WIdN3eDeJ0ZhSIZ4ZxHRkMg0kulkKRUqoVuBR4SCl1MnBuX4OUUq8AnQv1XgL8zv78O2BJVvtDSvMmUGyHvhqGiHgqjSMNymGRcjuxEraF0Mdkc6h1L64sd1BbKGm8F6VgJSDhcfRLvrY02l5jKRgMg0ouSsFp36A/BfztEI83UilVZX+uBkban8cAe7P67bPbDENEIpXGSoFyCmmXQ1sK0GdYanPLbgrDuq+ztBTL47X314NSiDRjJYSkJ5cEve1YebrSmme6UQoGw2CSi1K4HXgO2K6UeltEJgPbDvXAdshrv+syiMgXRWS1iKyuq6s7VDEMNomkwpnSlkLa7WxXCn1YCi2BAxTY69QcJSU4fLpwTryn6mvRFlxxIeXrX05FR14+4vXinjC+784Gg+GgyaVG8+PoZHht33cAlx3k8WpEpFIpVWVbH7V2+35gXFa/sXZbd/L8Gvg1wIIFC0yxnwEinkrhSEPaYZG2nDjDdjaTPsJSm0M1FGYrBY9WCsl0z+4jZwJUoafrtl5wlI3A6/Egjv65nQwGQ//IZaL5x/ZEs0tEXhSROhH5zEEe7yngWvvztcCTWe3X2FFIpwAtWW4mwxAQS6RxpgCnA+V25e4+CtdRGFHg92F5PDh92s2jy3l2VQoqpJPnKb+/X/JVfv/7jPnZT/s1xmAw9J9c3Efn2xPNH0OnvJgKfLOvQSLyKPAGMENE9onI54A7gfNEZBt6svpOu/szwA5gO3Af8NV+nofhEIklE7ZSsFAel05qB326j5pjTRSGtZUA4LDrLieVq9ux8YZtuiCP39dlW2+4KitxjR7drzEGg6H/5DLb19bno8DjSqmWXEIJlVJX9rDpnG76KuD6HGQxDBKReARfCnA6we3GlVAoQPqwFFpjrRSH9SQzgNPrIw0kcXdrKYQad+CLQSyvf5aCwWAYGnKxFP4mIpuBk4AXRaQciA6uWIahJp4Ia0vB5QCPG1cKktC3pZAIUBxpVwoev15tnOhOKaRThJt34Y3nvprZYDAMLbnUaL4FOA1YYGdKDaPXFRiOImJxrRTE6UQ8btwJiEkf+Y+UojkVoTAMzmLtPsrL1+8xnJDopBRa9hJMJXGnwJlvUlUYDIcjuUw0+9E+/rZVxqOBBYMplGHoiSciGaVgeX24kxBy5/c+0RxrpQVFQVjhsC0Fv60U4ilHV0uhfjsRe/WzK88oBYPhcCQX99Fv0UV1TrO/7wfuGDSJDMNCMhbCocByOXF5/XiSECwoh+bdPQ8KVBNJWbiS7RPNbrtwTiIp0HoAHvsMPPcdqNkEDduJpnRIqSvf1EQwGA5HcplonqKU+rSIXAmglApLrklrDEcMSdtNJG4XTp/294dLpsKBtT0PClShYvq5wllqRx8VagtAxQVqN0H9VkBg9W9h2nlE0RPMPdVnNhgMw0suSiEuIj7s1cciMgWIDapUhiEnFdUr0BxuN27btRPOHwfbnodQPeR1U9gmUI0V1c8Hbe4j8XpJOkDF7MVvF94Jkz4M/3sKbFpO1D0DCODtoT6z4egmkUiwb98+olETqzIUeL1exo4di8vlynlMLkrhe8DfgXEi8ghwOrD0oCQ0HLakbUvB4XLjKSxGARGnXS7zwFqY1k0OxEAVjqgACoc90SwixLxOSHvhnO/Bws+DCMy+FDY8QcIqAAL4jFI4Jtm3bx8FBQVMnDgx5yy5hoNDKUVDQwP79u1j0qRJOY/LJc3FP0TkHeAUQICvK6XqD15Uw1Dw9af/lw1VdYxInZ9T/4KqKs4CLI8HT1EJUSAqdtho1bvdKoVk6wE8Ea0U2txHAAm/C0dMwRnfaO/84W/Cxj+TtPeZa31mw9FFNBo1CmGIEBFGjBhBf3PE5Vpkxws0Aa3ALBH5cD/lMwwhT67dz0v7H6PW9QQhx3osoc9XuUe7e5xuD75i7SqKhyJQOhmq3uv2OKHAfgrtcgttE80ASZ8bZzjesXPFTPjyShIenRjXzCkcuxiFMHQczLXu01IQkbuATwMbAdtRjAJe6ffRjmE2Vu/jX57+FkXBz2KpwV3Nu6OhGd+URhQQyn+YP3zibxR5er8Jv/PM6wBYbg95xeU0AYlAC1TOg32rux0TCFZTGFYoy8IqbI8mSuV7cQfCXQeMnE002Aq01102GIaShoYGzjlHJ1Worq7G4XBQXl4OwKpVq3C7+5e9d/PmzVx33XW88847/PCHP+Tmm28ecJmHmlzmFJYAM5RSZnL5EHhqy+uEneuoLKihwjl7UI81YVQTr8fgk60BHi+ENftWcvaUj/Y6JhXTE3+Wx4OnsBiAZGsrzJgHG/8M4Ubwl3YYEwrXURCBdFFehycSlefDXZfs9jgJWymYFc2G4WDEiBGsXbsWgNtuu438/PxDupGXlpbyi1/8guXLlw+MgIcBuSiFHYALE3F0SBwI6GmYr541nvMnDe7av6fXbcB6OM1F4bE8sbCZln2roC+lENdKweH2YRXo6KNUMKAtBYAD78LUrLRVShGINFAYLobijmsOJD8ff1QRS8XwODqmyE4FAqScgvTzicxgGCxefPFFbr75ZpLJJAsXLuTee+/F4/EwceJEPvWpT/Hss8/i8/n4wx/+wNSpUzuMraiooKKigqeffnqYpB94clEKYWCtiLxIlmJQSt04aFIdhdSGdGVSJfE+eh4626vXcOnrafIbGvjXfULgc92WpuhARil4vBmlQDAMlXP156q1HZVCpImgSlMQVljlxR32ZRXk449Bdaiaa569hu+f9n0Wj1sMgLspRLTIZ/zKBr7/141sOtA6oPucNbqQ7308d0s8Go2ydOlSXnzxRaZPn84111zDvffey0033QRAUVER69ev56GHHuKmm27ib3871OKThz+5TDQ/BfwAeB1YY7+6dzIbeqQh0gRANDn48dk76rdS2ahwjR/HKVsUjg17+hyTimt97/D4sNxuEk6BUBh8xVAyqesitkAVAUt03qPSjm4lR2EhvjhsrFlHY7SRFXtX6GOkU/hbYiRLTIoLw+FBKpVi0qRJTJ8+HYBrr72WV15pny698sorM+9vvPHGsMg41ORiKRQrpX6e3SAiXx8keY5aWuPN4IFIMtKvcQeCB3iv7j0umnRR7sfaX4tDCcWXXkbdPfeQamrpc0w6oxT0JHjc60BCtqyj58H+NR0HNO4kZFlURMBV2jG81G3PSew8sBGAdfXrAGiKNVEcVDDFrFEw0K8n+uEi26I9VqzbXCyFa7tpWzrAchz1RFL6xtxfS+HhTQ/z76/8O6HOGUd7IJwI46nTLir/ooUAqEDfx0zF9Zi2cpoJnwtH2PYWVs6D5j16srmNXa8StNzkR8BbVtFhX+4ifdPfW7WF43el2V27jVAiRH2knpIgOMq7WR1tMAwDDoeDXbt2sX37dgAefvhhzjzzzMz2xx57LPN+6qmnDouMQ02PloKd6+gqYJKIPJW1qQBo7H6UoTsC0QTjm2r53q+SbLhjGxyf+9hdrbsy77NH9P1ktaNuPePqFcoSvMcfT8oCOq8Z6IZ0Qvdx2fUQkj43rrZxo+fp96q1MOVs/fmDl4h6x2DRgGdEeYd9eYtGkAai27by3UfTPHYGbPjoBpLRCKVRSI6s7FMeg2Eo8Hq9/Pa3v+WTn/xkZqL5y1/+cmZ7U1MTJ5xwAh6Ph0cffbTL+OrqahYsWEBrayuWZXHPPfewadMmCguP3ISPvbmPXgeqgDLgZ1ntAWDdYAp1tFHVEmVSQ5CCKDh2fNCvsbtadgGws2VnTkphf9VqxtWBVJZiud1E/RaOSBKSMXB6ehzXphScXh0qmsrz4Araaw3aJpsPrNVKofUA1G8hOepMoKHDwjUAf0k5QWDELj2Pcur7adbVrWN00E0pkDdybK6nbzAMGrfddlvm87vvvtttn29+85vcddddPe5j1KhR7Nu3b6BFG1Z6dB8ppXYrpVYopU5VSr2c9XpHKdV9ELqhW/Y3R8iP6ptuOph7tEU8FedA6ADQrhz6or5pB+PqFN5pOnQunufGFbX0jbwXVEL/SR1evagsnefDE03pjb4SGDEVtv4dlIIdL+s+Ke1q6jzRnFesLYdJNfr7+HrYs/51QtV7ASgcMyGnczEYDENPj0pBRFba7wERac16BURkYOPIjnL2NLaQH9GLwdOh3stbZrM3sJe00uN2tuzMaUxD4x5GNUP+cfMASBV48USAlt6fZlSb+8hnRwbl+fHG0qTStmI4+cuw9y3YsQK2/wP8ZRDW27pYCqW2UqhWpC1BCeS9uo5wjVZMeaOMpWA4/Nm1axdlZcfe/FdvE81XAyilCpRShVmvAqXUkeswGwZ2NdVSaHtiVDj36KO2+YTyZJKddRtyGpPYWwuA57hZAKSLCvBFIdXce1hqm6XgtC0Fyc/DH4Nw0hZ8/jVQOBaeuA42/AlmL0Fa9eS3o6SjpeAq0D+P0iDEKoqIzZrEvI1h9u7UOZSc5R3nIAwGw+FDb0rhL20fRORPQyDLUcvelnqK2pRCP/LI72rZRWFIcfnWOHtCVe1P7b2g6nT5TPcE7aKxSkopjECgeUfv4+x6Ci6vDkl15OfjjUGwrRyn0wOLvwWRJjj1a3DhXTgySqG4w76srLxG6dHlVJz/USbUQcnuJtICzhEmQ6rBcLjS20RzdlDu5MEW5EjmLxvf5KdvPMiI6FVIN3p2R2AvHw0pACSW+3TM7qbtfPLNFIvfdvPA1DRVW/7K2OOW9D6oRYeROit0mKijpJT8CDQ376a4l2Eqqm/wLjsk1VFQiAWEWuqhYLTuNP8amHoeFOroIXdrlLjPidUpZYU4ncTcFp54GvfYcZSediat//U/nLxFES3wIA5HztfAYDAMLb1ZCqqHz4ZOLN/yT1pdr1GQF6Mkz93lNWkUGUtB4mlI5OZC2tWwiSl1aUTBuDrYtemJ3gcohTOYJuUUHMXFgF5Y5kxDa33vcwrxmJap7QbvKtRZVUPNnXKxF7aHk3qCMRL53m73F/Pr5438iVPwzjqOpN9DfhTixYObIdZgMBwavVkKc+0JZQF8WZPLAigzr9BOS0wvTLvj0qlMKupa4ejRzR9QeI/+bCXsgvYjpvS5393BA4y0V4SMr1PsKN/Ch3rpHw9UkR8U4kXuzOpLT6meKAvWV/U8MNJEJJEiJQ6wn+LddmW0SHNDt0OUUviCCRJFxd1uT/hc0BynZPJMxOHAfdKJpF99k3SpqaNgGD4GOnU2wOLFi6mqqsLn01b2888/T0VFxwWdDz74INdddx3/+Mc/OPdcXbBq+fLlfOITn+Dxxx/n8ssvP5TTGlB6VApKKWPj50gwoSOKWuPdB2W1tNSQZy8OdiSBlr19KoVwIkxrPEJ+qzbmptQJ+6P12spw+bodU1+3kZKgIp2VW8g/YiQJINzUANFW8Hajy+u3Y4UsQsWujDLxFpWggFhL9+sUw8kwBWGFGtt9HiOV7wdC+MZPBKD8Q4upefVNRk84/FMbGI5eBjp1dhuPPPIICxb0nv14zpw5LFu2LKMUHn30UebOnXvIxx5ocq28ZuiFcFJPxrbGulcK4erdmc+OhPQZHqr31cKoJhDbcTe5wUGtw4Lq9T2OqWvYQmkAHBXt0T355Xo+IBp3QHUPaw7rt+IPCNHydoXhK9aTwbGWpm6HBOIBXXWtuHuDcczIaQC4xo0DwH/yyQDkVZpwVMPhxYsvvsiJJ57InDlz+Jd/+RdiMf0EN3HiRP793/+dOXPmsGjRokwqjIPljDPOYNWqVSQSCYLBINu3b2fevHmZ7WvWrOHMM8/kpJNO4oILLqCqSlv39913HwsXLmTu3LlcdtllhMPaF7106VJuvPFGTjvtNCZPnswTT/ThXs6RXBLiGfoglg6Bo2dLIVbX7rpxJchJKQQDBxjToDWCd9YsKndvpdZywP53YNyibsfUN+2kLACuMeMzbYXlo2kC4nFLl9Wc2NUBFarbRGkLpKeOyrT5i8oJA8keFtsF40EKwxDpFHnUhqdkBMmiIhx2Gm7P9OmUXnsNhRdc0Oe5G44Rnr2l14ecg2LUHLjozpy7D1Tq7Ouuuw6Hw8Fll13Grbfe2m3yPBHh3HPP5bnnnqOlpYWLL76YnTv1+qNEIsENN9zAk08+SXl5OY899hjf+c53eOCBB7j00kv5whe+AMCtt97K/fffzw033ABAVVUVK1euZPPmzVx88cUD4oYylsIAUNHYwKWvpWmNdp+NNNmoXTApl4UrAaqPNQMAwda9jLY9NwUXXIA/lCQUd3XNVppFQ+1u3CnIHzct0+Yu0U/8yZSva/prm/0171MaAPfY9qf4/BJtbSRbe1AKrfW4k+As6T7j6YjP/QuV//mjzHexLEb+x3/gPe64HuU3GIaagUid/cgjj7B+/XpeffVVXn31VR5++OEej3fFFVewbNkyli1bltk3wJYtW9iwYQPnnXce8+bN44477sikz9iwYQNnnHEGc+bM4ZFHHmHjxo2ZcUuWLMGyLGbNmkVNTc3BX4gsjKVwiKTSig9tbeTSlWne/PROmNVNp2btXoqVF+BJtJBs2Yurj/22BvYzukGhRhTis/2OBQ2QkjX0NNkTrNUL1wrHTMy0OQoLSQuQ9OqEdt1Qu28nI4D88e2Rxx4702k6GOh2TFP1bkYBvrJR3W73zpyJd+bMnk7PYOjXE/1w0Tl1diqV4qSTTgLg4osv5vbbb2fMmDEAFBQUcNVVV7Fq1Squueaabve3aNEi1q9fj9/vzygi0IEbs2fP7lbxLF26lOXLlzN37lwefPBBVqxYkdnm8Xg67GMgGBZLQUR2ich6EVkrIqvttlIR+YeIbLPfj4ik+83hOO64XlQWrevBLdSqfYDxkaV44hBu7DspXjBYy5gGhTV+NJ4Z+scztg4amndBDxZJrFE/1bsr28NGxeEg4nNA3AH12yCWdZOPBeG1X9Bcp+cNSie138TF5yNlQayludtj7dyj5ycqx0zvdrvBcCTQ39TZDoeDtWvXsnbtWm6//XaSyST19brUbiKR4G9/+xvHH997GuQ777yTH/3oRx3aZsyYQV1dXUYpJBKJjEUQCASorKwkkUjwyCOPDMyJ98JwWgpnKaXqs77fAryolLpTRG6xv39reETLnfpgFG9Ma+h4Y12X7cl0EmcwQdpSqLJSvNt2Eg1WUxQLgKfnCmTBUC1TGsBz+lScJSWkSgsZWx+gZqyDippNMKFrbvdUSxQQnBUjO7TH8t04ogpQULUOJp6uN/z5C7DlGcJqKhCmbFK7mSMixPPcBGv2o5Tq4iOt26HTbvhGjsnhKhkMhyeHmjo7FotxwQUXkEgkSKVSnHvuuRn/f09cdFHXgllut5snnniCG2+8kZaWFpLJJDfddBOzZ8/mBz/4ASeffDLl5eWcfPLJBALdW+8DxeHkProEWGx//h2wgiNAKexracJvlx1IdvNU3RBpoCgMSb8geX48cYiK6Kf2MfN73G+ouRZ/HHwT9fyANXE8Y6s3UOt0QO3Grkoh0oSEFCC4KjrmFkoW+LCCAZQ7H1n5XzDhNJ3xdMszcO5tpP70T1LWdtyjOrqCEjMnMnH7VvYE9jChsD2zaSqdwr1pFymXA+8MYykYjkwGInV2Xl4ea9b0PM/XxtKlS1m6dGmX9gcffDDzed68eR3mM9r4yle+wle+8pVexwIEg7kn2+yN4ZpoVsDzIrJGRL5ot41USrWF6VQDI7sbKCJfFJHVIrK6rq7rk/lQUxVoxG+nM0oHuv5R6qP1FIZA/A4cPj/eBIRFoH5rr/tNNDUD4C3Tl8E/bTpjG6DGnQ81m7oOaNyJK2QRzXcinRbgeMsqyG9NsPv0r+oMpy/cBk/fDOUz4dSv4ahpIFjsQZwdnxFKTz+T0U3w7voXOrTvaNnB1D1xYjPGdzmWwWA4shkupfAhpdR84CLgehH5cPZGpWdMup01UUr9Wim1QCm1oPwwyLZZFWjEb7uPVKhrsrv6cD1FYYUj340jLx9LQQwn1G3pdb/JVq1gnKV6aqVg+iz8MWi2RkBtV6WQbPyAvBAkSvK7bCuZM5/RDfBuwTgdsvfaPQA0XXQnK6vfwlcf7LBGoY2xiz8CQO2rHZXChn1rmFgDBSct7PUcDIYjGZM6ewhRSu2332vR2VgXATUiUglgv9cOh2z9pS7cQp6tC6xI12R39eE6SoLgHVGCM0/fsCO+yj4thWRQ5yJqy2HknaKL5iQDHm0pdIo0aKjfTGkAVEXXDKSjTzkbC9j39itw1eOw9Bm4aR3f3/0kX3nhK5Q0JVCjuv74vdOnEylw4353c4fIhgNvv4IzDaNOWdzrORgMhiOPIVcKIpInIgVtn4HzgQ3AU8C1drdrgSeHWraDoSHcjL8thYVdSCebuoYtFAfBP3o8blspxD0VfVoKhBJ6n/Y6AM9UnRbDakhBrKXLAri6pg/0auaRXUNEffPmogSS723QCe0mns6uwF62rXmRu5+tYEQAps46vcs4sSyS82Yy7YMo/7biG5kcT6m1esFR3vye50QMBsORyXBYCiOBlSLyHrAKeFop9XfgTuA8EdkGnGt/P+xpjTRmlII7BqlkvMP24J6tOBR4J8zAla9dNHF3KTTthFSi+52mkli2gmlTCo4RI4jmucirtc2STi6kuqY9FEbAV9k1GsiRn09kfAWVO1qpCuppm4c3PsRXn04zbkeAkquvZtSVn+1WlGkXfYrSIPieeIE73ryD6lA1ZdsbCI8rw1FkktsZDEcbQ64UlFI7lFJz7ddspdQP7fYGpdQ5SqlpSqlzlVLdZ2I7zEiEa/DYXqO8KASbO5bNjFXpJ3rn5OPx5OubaMIqgHQSGnsofBNuwBEVUhZYeXmADhENjymhuDaqJ1s6pQdotheu5Y8eT3f4589n2gHFyr2v0Bht5IMX/sLkqjQjb76ZUf/vVlydsjq2UXzJEgouvJDPvJgk8uzz/GPrM8zao8hb1H2qDYPBcGRj0lwcIo5g+1KLvCi0dlqYlmxoBsA5dlJGKSSxs5z25EIK1uCOCvE8V4f1AcnxlYyuS9NaeQK89X/QagdrpRKEW3SRnKKx3ddDqjxlMf44vPTKQyzbvIyProxB+QiKlizp9fzE4WD0j+8iPX0SS16J8+pf78WThDHnX9zrOIPhcKWmpoarrrqKyZMnc9JJJ3Hqqafyl7/8pe+B/WTz5s2ceuqpeDwefvrTnw74/geLw2mdwmHNb1a9gE9NxGV5OrQ7Iu1ZRPOjitZOloLVot09rpEVeJV2FyXT9mWv70kp1OKJQiK/47E8kydT+MJ7HFjwdYr+fgP86XNw7V+hZS+xsNbv3lHdLybLW7QIZQlTX9nJqqpf8609iopbPt+lalp3WG43oz9zHdZ3v8tHXg6SdDvIs7OeGgxHEkoplixZwrXXXssf/vAHAHbv3s1TTz014McqLS3lF7/4BcuXLx/wfQ8mRinkwLrqvdyz6RvEqj9Bormj2+TDLv2Ervwe/LEYLa3tE8AqlcQdVKQtPSfgtlciJmNxKBwLdd1HIKWC1fgikCrsWKWs0F7IVtcU4LiL7oKnboBtz4PDRSqiMyK5RnbvBnJVVlJ4+WWc//gTLNgWIz12FCVXXZXzNSi86EL2/+D7zNifIrJoBpa3+4prBsPhzD//+U/cbneHVcsTJkzIZB1NpVLccsstrFixglgsxvXXX8+XvvQlVqxYwW233UZZWRkbNmzgpJNO4ve//3232VDbqKiooKKigqeffnrQz2sgMUohB9Ye2IWI4pL5BfzrorM7bHvg/+4GQCoryN+3l/rAgcy2QN1GigKQKPQiloXDnh9Ih8IwenqPlkKwdR+FYVCjO07kVkw5njqgZdc2OOv78NKPYNV94PRghR0kXRZWL5O/o276V5r+9lfKW2OM/en3c7IS2nAUFOA9+0wSz/2TMedfkvM4g6En7lp1F5sbNw/oPmeWzuRbi3pOhLBx40bm9xI1d//991NUVMTbb79NLBbj9NNP5/zzzwf0queNGzcyevRoTj/9dF577TU+9KHeaiEemRilkAPb6vWNPj8vwZjiTlXPIrZ7aOxY/B/spbWmPa1t/b63KAmCKtM3asuvn/xVJAJl0+GdhyCdBqvj1E6gdgP5EVD2GoU2iidOpw6I7t8LDhectBRW/CcA7sh4IiWeXp9cnKWlTPzpfxHbuoWCD3+4x349MXrp5znw/nZKz7uw32MNhsOR66+/npUrV+J2u3n77bd5/vnnWbduXaZgTUtLC9u2bcPtdrNo0SLG2unl582bx65du4xSOFbZ3VwNQEJ1k1vEXsXsmzCZBG8QbqqG5r1QPI66nS9RGlA4ZukfkmXXcCUS1UohEYbW/VA8rn1/VesIbH2OgkgZ4U61CpxFRUS8Fmq/loeTlsIrPyHlLyMvkCJZ2nfZ7IKzz6Lg7LP6dwFs/CeeyNTnnzuosQZDZ3p7oh8sZs+ezZ/+9KfM91/+8pfU19dnSmkqpfjv//5vLuhUDGrFihUd0lQ7HA6Sya6LVY8GTPRRDlQFdY6lbsttRvTksXecThgXTThg23OQSlB94G1KA+C1S1CK00nCKVoplM/Q47NdSErBc98maBXiUOAq7bo6OVTmx1XbrL8UjIKP/4Kmj/+UkoBClZcOzAkbDEcpZ599NtFolHvvvTfT1lbeEuCCCy7g3nvvJZHQ/9dbt24lFAoNuZzDiVEKOdAY1WGnzeGOlY2UUljRFApwj9a1kFulALY+B7tfY1c8gT8OJeOmZMYk3BYSjUGZrRSyJ5sPvAO7XiU8Qbtn3N0ohURFCfkN4fa0EydeTd2IKZQGwDWy2xyCBoPBRkRYvnw5L7/8MpMmTWLRokVce+21mUyon//855k1axbz58/n+OOP50tf+lKfFsF3v/vdbqOXqqurGTt2LHfffTd33HEHY8eOpbWHSoaHE8Z91AfRRApJa3dNS9YkMkBLrAVvTJHytOcoqk37YMfLEAtQk9DmpntUe9GbpMepLYW8MvCVdLQU1j0ODjcx/yQAfCO63uRl9CjK1u+lJdZCsVcfs75mF2UpsHoIRzUYDO1UVlaybNmybrdZlsWPfvSjLkVwFi9ezOLFizPf/+d//ifz+fbbb+92X6NGjcqU1DySMJZCH+xuCJOfauS+nycZu6Wjlq8OV+sUFx4HjkLtzw9EIsTKpsGeN2hN6DkEZ1aYqPJ5SIZCIKLnFarWabdRKgkbnoDpFxCzM6TmlVXSGd/Y8XgTcGDf+5m2zVtfA6B8gil/aTAYDg2jFPpgZ32QUS0hisIw8kCqQ7bQ6lA1eVFw+FxYhTrCyB9RbP/Urwl+8WW8DdrsdI9rn0h2+P1Y0TiBeACOu1i7jNb9EXaugFAdzPkUyWad4aOgvKtSKLLXKtTu0FFOSinef38lAIVjJg38BTAYDMcUxn3UiarWAOl0+2V5b28zJUGd8a4wpAjHA+R5tFVQHarGFwOX34ujSLflRWFL4xaSxVOYuVeRKivGmVXRzFlYTMGBava07mH2KV+B95+Cp78BlhO8xTDtfFKP63A4z4iu9SIqpsymFtiw/p/sH+djTtkcJq6vI+12ZjKpGgwGw8FilAKwp7GZWMLBT15+htfD/0l415dJR+2neyvGZ1q1dVAcgpa6TeSNPQWA6ta9zIgqPKPysbxexO2mOI5ekKMUM/cp3KfO67B2wDd1GmPXb2Z3805ml82GJffCo1dC5QlwyldRTg91VR+QsgQrv2vBnOKJM6gFRrywlv1r1/Lwhwv50UaF/4LzMi4sg8FgOFiOeaXw368/w6+2fptY9cdxlbyJw5vi8lNhQekJADTE9pP4nzaloGip3cBoWynUNG7jpBg4SvSEsHvKFBbu2MX99ZspaIxyXABKF57W4XgjZs2j/o9/pWbHBpj6MRgxBb62KrN9c8P7TN7YSHziqG4XojkKCqCslHk7G5m3E059vwVfHEZe1X3qa4PBYOgPx7RSCMWS3L/uYcSXwlu5PNNeURrkUwu1pbCmppZtzbq9OAgtWdFC1YE95MUUjnK9DmHE5z9H7N9upuC1DTR7dcRSvr0opo38mbOoB8JbNuvyQp14a/mvOLkOim74fI9yT33sjwAEV74G3/se1tRJ+ObN69/JGwwGQzcckxPNL3+wjSUPX8u1D/+V/NQGfvB8irNC+Xy4/ESmFE1hb2Bvpm996x4qWtrdR81ZqbHrgg14YuAYqUNBCy+8kMS4Ci55OcK4NftIeJ14pk/vcGz3VD1RLDv2dJErlU5R+OcVhIrcVC75ZI/yu8aMwTVmDCWf/hRjfv5zJvz4p72mtzAYDO0MVersBx98EBHhhRfaa5wvX74cEcmk0TgcOSaVwu7Xf8nZz7zNntT3+PzzKWasUdxwfyM/e+kfjBM3e4NZSqF+K+UtgMuBNwHBRh13nFZpHHURLCW4J+saBuJwMPGW7zK22eLkLYqi+QsRh6PDsR35eYTK8sjb29BFrjXvPM1xH8RIfOI8JMdkdYUXnI931qyDvBIGw7FFW+rsD3/4w+zYsYM1a9awbNmyQVtPMGfOnA5rIh599FHmzp07KMcaKI5JpbCk+EzOXav42f0JffM+/3SUu5Tdfy/h9GfWsK95Vyb0NLR7O840eGboyJ5IXR1EmmkM1VFZr/t4pk3L7LvwnHOYtmIFo267jZHf6j63S3zCKEZVxzM1j9vY9uTvATj+yq8O+DkbDIbcUmd/85vfZOHChZxwwgn86le/AnTuo8WLF3P55Zczc+ZMrr766g7h6T1xxhlnsGrVKhKJBMFgkO3btzMvy9W7Zs0azjzzTE466SQuuOACqqp04az77ruPhQsXMnfuXC677LJMKo6lS5dy4403ctpppzF58uRBsTiOyTmFwo9/gvEFhVg3fR3n2EIqf3Yvybo6qr79H8x9eRXTR8VoCFVTll9Jw15dNCdv/gJiG7YSjwAb/0JNUQXj6hXKEjwTJ3bYv7O8nJIrPt3j8V3TpjL63Q/YWb+NeWP0nEMkGaHgjU00ji/muAndV08zGI4mqn/0I2LvD2zqbM9xMxn17W/3uH2oU2eLCOeeey7PPfccLS0tXHzxxezcqe8piUSCG264gSeffJLy8nIee+wxvvOd7/DAAw9w6aWX8oUvfAGAW2+9lfvvvz+juKqqqli5ciWbN2/m4osv5vLLL+/XNeqLY9JSAMhffA5TX3qVScufR1wuXKNHM/ZXv0Y5HZywU7H3xf/Hmtd/yv5qrbn9C3XEUSpVAGv/QHX1O4yrA8ZU5OzqaaPyhJNxpuGV1x/NtK187ymm7UvhP+fgMpgaDIb+c/311zN37lwWLlwIwPPPP89DDz3EvHnzOPnkk2loaGDbtm0AmdTZlmVlUmfnwhVXXMGyZctYtmwZV155ZaZ9y5YtbNiwgfPOO4958+Zxxx13ZNxYGzZs4IwzzmDOnDk88sgjbNzYnpJ/yZIlWJbFrFmzqKmp6XK8Q+WYtBTacHZKTW15PDhOmMWsPevZte0Zfl/zKh9qtEDAe4LtB0wXwb5VVEf3MK7Owruw//78igWnEwCaXvwHDR9roC5Sx6vL7uZKYMaSaw79xAyGI4DenugHi+FInb1o0SLWr1+P3+9nelbgiVKK2bNn88Ybb3QZs3TpUpYvX87cuXN58MEHWbFiRWZbthy5uLD6yzFrKfRE0cmnMakGfjdyBgV7nJy3uQD3hIk4y8tIWYIj6gCnl80OHxXNUDDz+H4fwz1+PNaieZy9JsFXn/sStz5wNRe/EIApE/FNnzHwJ2UwGIDhS5195513dkmyN2PGDOrq6jJKIZFIZCyCQCBAZWUliUSCRx555JCP3x+MUuhEwcmnYClY/EwN33oijb9iNGN+fg9iWUQK3bhbokSvf4utQScW4Jk+rc99dsfopV+gLABnPr6NW/+QoKCwnCm/us+ElhoMg8hQps7O5qKLLuKsszq6ht1uN0888QTf+ta3mDt3LvPmzeP1118H4Ac/+AEnn3wyp59+OjNnDm2iSxkM82OoWLBggVq9evWA7jMdibBpwUk4UgqZNonpT/wFyzbXXrvgNPa7g+T9/D959pf/zvVPp5n87DN4JvU/EZ1Kpfjg/AtI7N+Pd84cxtz9sw6J8wyGo5H333+f4447brjFOKbo4Zr3+PR5TM8pdIfl88Gs6ajNHzD57p9nFAKAf9QY8nZt4PZXv8v31lpYhfm4x48/qOOIw8Hon/yExN49FH7sY13WMxgMBsNwYJRCN0z94Y9Jt7Z2WH8AMHLcDFxrN/P1xyJM2q+o/PkPDulm7p9/Isw/8VDFNRgMhgHDKIVu8HZKTdFG8aWXEdu6jRPXrSPv6k9TeEE3yYsMBoPhCMYohX7gn38ik/74GKnmZqyiouEWx2A4IlFKmYCKIeJg5oxN9NFB4CguNj9qg+Eg8Hq9NDQ0DEp8vaEjSikaGhrwer39GmcsBYPBMGSMHTuWffv2UVdXN9yiHBN4vV7Gjh3brzGHnVIQkQuBnwMO4DdKqTuHWSSDwTBAuFwuJh1ECLdh6Dis3Eci4gB+CVwEzAKuFBGTF9pgMBiGiMNKKQCLgO1KqR1KqTiwDLhkmGUyGAyGY4bDTSmMAfZmfd9nt2UQkS+KyGoRWW38kgaDwTCwHHZzCn2hlPo18GsAEakTkd0HsZsyoH5ABRsYjFz953CVzcjVPw5XueDwle1Q5Pq7UurC7jYcbkphP5CdAGis3dYtSqnygzmIiKxWSi04mLGDiZGr/xyushm5+sfhKhccvrINllyHm/vobWCaiEwSETdwBdB7+kGDwWAwDBiHlaWglEqKyNeA59AhqQ8opTb2McxgMBgMA8RhpRQAlFLPAM8M8mF+Pcj7P1iMXP3ncJXNyNU/Dle54PCVbVDkOqLrKRgMBoNhYDnc5hQMBoPBMIwcU0pBRC4UkS0isl1EbhlmWcaJyEsisklENorI1+3220Rkv4istV8fGQbZdonIevv4q+22UhH5h4hss99LhlimGVnXZK2ItIrITcN1vUTkARGpFZENWW3dXiPR/ML+3a0TkflDLNdPRGSzfey/iEix3T5RRCJZ1+7/hliuHv92IvIf9vXaIiIXDLFcj2XJtEtE1trtQ3m9ero/DP5vTCl1TLzQE9cfAJMBN/AeMGsY5akE5tufC4Ct6NQetwE3D/O12gWUdWr7MXCL/fkW4K5h/ltWAxOG63oBHwbmAxv6ukbAR4Bn0SUQTwHeGmK5zgec9ue7suSamN1vGK5Xt387+//gPcADTLL/bx1DJVen7T8DvjsM16un+8Og/8aOJUvhsEqhoZSqUkq9Y38OAO/TafX2YcYlwO/sz78DlgyfKJwDfKCUOpiFiwOCUuoVoLFTc0/X6BLgIaV5EygWkcqhkksp9bxSqq36/Jvo9T9DSg/XqycuAZYppWJKqZ3AdvT/75DKJSICfAp4dDCO3Ru93B8G/Td2LCmFPlNoDBciMhE4EXjLbvqabQI+MNRuGhsFPC8ia0Tki3bbSKVUlf25Ghg5DHK1cQUd/1GH+3q10dM1Opx+e/+CfqJsY5KIvCsiL4vIGcMgT3d/u8Plep0B1CiltmW1Dfn16nR/GPTf2LGkFA5LRCQf+BNwk1KqFbgXmALMA6rQ5utQ8yGl1Hx0ttrrReTD2RuVtleHJWxN9KLGi4HH7abD4Xp1YTivUU+IyHeAJPCI3VQFjFdKnQh8A/iDiBQOoUiH5d8uiyvp+PAx5Nerm/tDhsH6jR1LSqFfKTSGAhFxof/gjyil/gyglKpRSqWUUmngPgbJbO4NpdR++70W+IstQ02bOWq/1w61XDYXAe8opWpsGYf9emXR0zUa9t+eiCwFPgZcbd9MsN0zDfbnNWjfffcFygeBXv52h8P1cgKXAo+1tQ319eru/sAQ/MaOJaVwWKXQsP2V9wPvK6XuzmrP9gN+AtjQeewgy5UnIgVtn9GTlBvQ1+pau9u1wJNDKVcWHZ7ehvt6daKna/QUcI0dIXIK0JLlAhh0RBeu+nfgYqVUOKu9XHQNE0RkMjAN2DGEcvX0t3sKuEJEPCIyyZZr1VDJZXMusFkpta+tYSivV0/3B4biNzYUM+mHyws9Q78VreG/M8yyfAht+q0D1tqvjwAPA+vt9qeAyiGWazI68uM9YGPbdQJGAC8C24AXgNJhuGZ5QANQlNU2LNcLrZiqgATaf/u5nq4ROiLkl/bvbj2wYIjl2o72N7f9zv7P7nuZ/TdeC7wDfHyI5erxbwd8x75eW4CLhlIuu/1B4Mud+g7l9erp/jDovzGzotlgMBgMGY4l95HBYDAY+sAoBYPBYDBkMErBYDAYDBmMUjAYDAZDBqMUDAaDwZDBKAXDkCMiSkR+lvX9ZhG5bYD2/aCIXD4Q++rjOJ8UkfdF5KVutk0Tkb+JyAd2qpCXOq8KH0pEZImIzMr6fruInDtc8hgOb4xSMAwHMeBSESkbbkGysVex5srngC8opc7qtA8v8DTwa6XUFKXUScAN6PUfg0bboqoeWILOsAmAUuq7SqkXBlMew5GLUQqG4SCJLiX4r503dH7SF5Gg/b7YTkL2pIjsEJE7ReRqEVkluvbDlKzdnCsiq0Vkq4h8zB7vEF1X4G07AduXsvb7qog8BWzqRp4r7f1vEJG77LbvohcX3S8iP+k05GrgDaVUZrW8UmqDUupBe2yenfxtlZ1Y7RK7famI/FlE/i46V/6Ps2Q4X0TeEJF3RORxOx9OW92Lu0TkHeCTIvIF+/zeE5E/iYhfRE5D54r6iegaAFOyr7GInGPLsd6Wy5O17+/bx1wvIjPt9jOlvZ7Au22r3w1HD0YpGIaLXwJXi0hRP8bMBb4MHAd8FpiulFoE/Ab9NN7GRHQenY8C/2c/vX8OvfR/IbAQ+IKdQgF0Pv2vK6U65LERkdHo+gNno5O2LRSRJUqp24HV6DxC3+wk42z0atee+A7wT1vus9A36zx72zzg08Ac4NOiC62UAbcC5yqdpHA1OhlbGw1KqflKqWXAn5VSC5VSc9Gplj+nlHodvVr4m0qpeUqpD7LOz4teuftppdQcdM32r2Ttu94+5r3AzXbbzcD1Sql56CyikV7O1XAEYpSCYVhQOuPjQ8CN/Rj2ttJ55mPo5fzP2+3r0YqgjT8qpdJKpzzeAcxE53C6RnQVrbfQ6QKm2f1XKZ23vzMLgRVKqTql6xE8gi7KkjOiK51tEJG2hGbnA7fYcqwAvMB4e9uLSqkWpVQUbbVMQBdMmQW8Zo+51m5v47Gsz8fbVs96tMUyuw/xZgA7lVJb7e+/63R+bTKvof36vgbcLSI3AsWqvU6D4SihPz5Ug2GguQf9VP3brLYk9sOKiFjoKnltxLI+p7O+p+n4W+6cu0Whc8PcoJR6LnuDiCwGQgcjfA9sJOvGqpT6hIgsAH7adkjgMqXUlk5ynEzH80uhz0mAfyilruzheNmyPwgsUUq9Jzor6uKDPw3IkqdNFpRSd4rI0+g8PK+JyAVKqc2HeBzDYYSxFAzDhlKqEfgj2rXTxi7gJPvzxYDrIHb9SRGx7HmGyeikas8BXxGdjhgRmZ7ltumJVcCZIlJmT+ReCbzcx5g/AKeLyMVZbf6sz88BN4iI2HKc2Mf+3rT3N9XunyciPaVrLgCq7HO8Oqs9YG/rzBZgYtu+0S65Xs9PRKYopdYrpe5CZx6e2Yf8hiMMoxQMw83PgOwopPvQN+L3gFM5uKf4Pegb+rPoTJdR9LzDJuAd0UXaf0UflrLSqYdvAV5CZ41do5TqNWW4UiqCrlvwZXtC/A30nMAddpcfoBXdOhHZaH/vbX91wFLgURFZB7xBzzfi/4d2jb0GZD+9LwO+aU8MZybk7etyHfC47XJKA30Vo7/JdoetQ2cWfbaP/oYjDJMl1WAwGAwZjKVgMBgMhgxGKRgMBoMhg1EKBoPBYMhglILBYDAYMhilYDAYDIYMRikYDAaDIYNRCgaDwWDIYJSCwWAwGDL8fyPEtjHCVQZ0AAAAAElFTkSuQmCC", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAADgCAYAAADsbXoVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABl5klEQVR4nO2deZgcZZ34P9+qvuY+MpPJ5CIJckiABAi3CCqCqAusoCuyC1l3dUUUcVd3XWU98Pip64m6uCqKKAKKiojriaKCIDckQMIRckySSebs6Z7po47398dbVdM90z2ZHDOZTN7P88zTXVVvVb013V3f+t6ilMJgMBgMBgBrf0/AYDAYDDMHIxQMBoPBEGGEgsFgMBgijFAwGAwGQ4QRCgaDwWCIMELBYDAYDBFGKBhmJSJyloh07e95GAwHGkYoGPYKEXmLiDwsIlkR2S4ivxSRl+3F8ZSIvKRk+SwR8YPjZ0RkvYj8476ZfdU53BjM44Ix678YrF8dLK8WkXurHOMeEckH8+4VkZ+ISOcE53y9iDwoIsMi0iciN4vIwn16YbsguG537DxF5KMi8v1d7FcMrjX8e6Jk+z+JyLrg89shIv8nIg1TeS2GPccIBcMeIyL/CnwJ+BTQASwG/ge4YILdqh0rNsHmbUqpeqAR+A/gmyJy1G5PePd4FrhszPzeBLywG8d4VzDvw4Fm4IuVBonIxcAP0P/LNmA5UADuFZGWPZj7biMidcBFQBr4+z04xGeVUvUlfyuC456J/n5copRqAF4K3Lav5m3Y9xihYNgjRKQJuBa4Uin1E6XUsFLKUUr9XCn1/mDMSSJyv4gMBlrEV0UkUXIMJSJXishzwHMi8qdg0xPB0+bflZ5Tae4ABoCjRCQpIl8SkW3B35dEJFllvvNF5Mci0iMiL4rIVbu4xJ8DLyu5Kb8GeBLo3r3/FCil+oEfA0dXmJcAnwc+oZT6gVIqp5TqBv4ZyALvDcatFpH7gv9hOnjyflXJcZpE5Ibg/7xVRD4hInbJvveKyOdEZCC4/vPGTOUiYBD9mV6+u9c4AScC9yulHgv/F0qp7yqlMvvwHIZ9iBEKhj3lVCAF/HSCMR76ptYWjH8V8M4xYy4ETgaOUkq9PFi3InjaLHuiFBFLRP4W/dS9BvgQcAqwElgBnARcM3YSImKhb/JPAAuCeVwtIudOMPc88DPgzcHyZcBNE4yvioi0oW+6j1XYfARaw/pR6UqllI8WJK8uWX0yWlNpAz4C/EREWoNtNwIu8BLgOOActGAp3Xd9sO9ngRsCgRRyOXALcCtwpIicsLvXWYW/AueKyMdE5PRqQtswczBCwbCnzAF6lVJutQFKqUeUUg8opVyl1Ebgf4Ezxwz7f8HTY26Cc80XkUGgF30z/Ael1HrgUuBapdROpVQP8DHgHyrsfyLQrpS6VilVVEptAL7J6A2/GjcBl4lIczDvO3YxfizXBfN+AtgO/GuFMW3B6/YK27aXbAfYCXwp0MhuQ9/kXyciHcBrgasDjW0n2lRVen2blFLfVEp5wHeBTrTJDxFZDLwC+IFSagdwNyWms0nyvkAjDP++C6CU+jPwBuB44BdAn4h8IdRiDDOPiey4BsNE9AFtIhKrJhhE5HDgC8AqoBb9fXtkzLAtkzjXNqVUJafrfGBTyfKmYN1YDmFUsITYwJ8nOqlS6l4RaUdrJHcppXLlD9e75Cql1Ld2MaY3eO0EXhyzrbNkO8BWVV7BMrzeQ4A4sL1kfhbl/9vI7KWUGgnG1Qer/gF4Rin1eLB8M/B5EXmfUsopnZCIfBD4YLD4faXUO4L3n1NKjdPSgvP9EvhloLG9Aq0VrUc/JBhmGEZTMOwp96OdoRdOMOZ6YB1wmFKqEX0zGXtX3ZsyvdvQN8SQxcG6sWwBXlRKNZf8NSilXjuJc3wf+Df20HQ0CdYDXcAbS1cGN9CL0E/tIQvGmHzC692C/izaSq6vUSm1fJJzuAxYJiLdItKNFuRtaO2jDKXUp0qcye8Yu30ilFK+Uupu4PdU8K8YZgZGKBj2CKVUGvgw8DURuVBEakUkLiLnichng2ENwBCQFZEjgSsmcegdwLJJTuMW4BoRaQ/s9h9G38TH8iCQEZH/EJEaEbFF5GgROXES57gObdf/U5XtIiKp0r9Jzh3QznPgfcF1vCU4xjzgW+hoq9KIpbnAVcH/+Y3oSJ7/U0ptB36DfrpvDHwvhwaRPxMiIqcCh6L9MSuDv6PR0VC7a0KqdPwLROTNItIimpPQprgH9vbYhqnBCAXDHqOU+jzaTn4N0IN+Yn0Xo7b39wFvATJoG/5kQhE/Cnw3sEu/aRdjPwE8jI4KWgM8GqwbO08PeD36hvci2iTzLaBpV5MJ/B13jzHblHIakCv9k4nDayud4za0Cee9aLPc00ANcLpSqq9k6F+Bw4L5fxK4uGT7ZUAi2HcAuB1tftoVlwM/U0qtUUp1h3/Al4HXlziyd8W/S3meQmj2GgDeBjyHfkD4PvDfSqmbJ3lcwzQjpsmOwTDzEZ0w989KqT1ODDQYJoPRFAwGg8EQYYSCwWAwGCKM+chgMBgMEUZTMBgMBkOEEQoGg8FgiDigM5pf85rXqF/96lf7exoGg8FwoFE1Nf+A1hR6e3t3PchgMBgMk+aAFgoGg8Fg2LcYoWAwGAyGiCkTCiKySET+ICJPi8hTIvKeYH2riPxWRJ4LXluC9SIi14nI8yLypIgcP1VzMxgMBkNlptLR7AL/ppR6VHQ/1kdE5LfAauBupdSnReQDwAfQLRbPQ9d1OQzdEOT64HW3cByHrq4u8vn8ProMw0SkUikWLlxIPB7f31MxGAz7gCkTCkHlxu3B+4yIPIPuenUBcFYw7LvAPWihcAFwU1B47AERaRaRzuA4k6arq4uGhgaWLFnCbta+N+wmSin6+vro6upi6dKl+/TY3/zJB0hue4oj3fqqYzaS5X7pJ1ejFd55XpLXFtrLxmSLBbY/t5VDj1qCJaOKcRGfH9fsICceAIc+V2DTkgRuvPp35tgncsxvmcP8tuo14rZs24F6oX9S11gNLya4Nvi2gALbV4gPlg+Wr5D9kG/qxAU3Lng2WB7YnsL2wFIg+2lOU0GuxiLbYJFpsIg7ivqMT0PGJ+ZWvsChJpt1RyYZaq7cM6g+49G5zeW5I6o3nLMdxSkP5VlyzFKSduVbskKx4YkXiA+NtreQFcdy0Yf3fUX3aQlJFZEl6BaBfwU6Sm703QTdn9ACo7QpSFewrkwoiMjbgbcDLF68eNy58vm8EQjThIgwZ84cenp69ulxlVL8X++dtNku5/R6Vce9+HycNz0jfPRtQiZhcV9SuLTnRRr80TH3b49x/J8t+lof5SU1oxseTVn8riVFi6tY0Ofz+p8rvvca4YFjKn9vjtyoeMXdiqdfOsziFc9XndOOJxIc0gWZ2t2/bgBREPMh5kLMAyXgWeBbwauAsvauCcXuYimIu5BwIO6Ba4ETA8/W8/FFz1Md4D85UVCf09dYykgS8okK44GXZuDUv4zw/AL4v9OE9UvK/wkXPOFzzoNw9WFZnCoPHEduhFUP+GxvfpzDmv2KY9ZjsfL+GMNJKARz2dK2eTevcHJMuVAQkXp0r9mrlVJDpTdrpZQS2b1nDKXUN4BvAKxatarivkYgTB9T8b/esukFfPHpqVvAvI/+oeq4ure/jubhDdzW/gmeOGkO7/r9uxh+x485bO7KaEzho5cCj1J8xfuZ95q3R+uz638ID3ycH775d9T99Wm6vn0l/3rEu2n7p/EtH/xikRf/5nyKbEJUHfM++kzVOcVfeQw7j2rj7B9Vn/eBjFJqVv++lO/j9ffjdO/AqkkRnzcPq66u6nhn+3bSd/6c2C23cOQffQ77+B/Ltnc9eTUZfs1dF/6e+Lx5FY/x7E3X43Ed1hlXMu+iqyuO+dFvr+XsW29hwfv/g86/X72nlzcppjT6SETiaIFws1LqJ8HqHSLSGWzvRPedBdgKLCrZfWGw7oCir6+PlStXsnLlSubNm8eCBQui5WKxuNvHW7duHaeeeirJZJLPfe5zUzDjmceWNX9mYRfM7av81BTh6C6gAz/4AUsbtflqQ3pD2ZDMwA4AevrLu35uHtpM0k4yt3Yuzha9ze2pnPfSf8MNFDdtotiYIpYZoehV/hyzxSw1wy7x1sm2IDjwmM0CAUAsi1hbGzVHLyd56KETCgSAeGcnbf/ydlre/GbcnTvxR0bKtoffLW9goOoxUln9fRoZTlfc7iuf+17UDxk1dc2TvZQ9ZiqjjwS4Ad379Qslm+5EN/YgeP1ZyfrLgiikU4D07voTZgJz5szh8ccf5/HHH+cd73gH733ve6PlRKKCDroLWltbue6663jf+943BbOdmRQ3P8T59whn31P5RxLhaaGQf+opWl/sJ2kn2TBYLhQKaW3f7+8vf77YnNnMooZFWGJR3NIFgFvBDFbcsoXer/8vDa95DcXjX0rDsGLj0MaK09mU2UTjCNTO6ai43TB7SRyiTdnFLeUPH8Uu/d2aSCjEh7QgGRkZqrj9yZ4nSQ/p76ZVs1uN/faIqdQUTkd3k3qliDwe/L0W+DTwahF5Djg7WAb4P2AD8Dy6S9c7p3Bu08rdd9/NcccdxzHHHMNb3/pWCoUCAEuWLOHf//3fOeaYYzjppJN4/vnxtuq5c+dy4oknHlTRPU39a7RNveBOOE5cj8GmGFZtLUO33saSxiW8kH4h2j6QH8Aa0VFoQ4M7yvbdktnCogatmBa3aNusWyFDfscnP4XYNh3/+QEa5i6gMQcvDL4wbhzA5p7nSTnQ2L5g8hdrmBXEFwVCYdOmaJ2XTuMP6Ru9O4FQUAP64aeQy1Tc/uuNv6be05Z+SVZ3WO8rpjL66F6q19d4VYXxCrhyX87hYz9/iqe3VZa+e8pR8xv5yN9Mth+6dnyvXr2au+++m8MPP5zLLruM66+/nquvvhqApqYm1qxZw0033cTVV1/NXXfdtU/ne6AxOJznJe5zPOq1YBc9fOWXRQ2V4XrkamM0nnU+6R//hJeecRYPpddHmzekN1Cr5S/ZodGulr7y6cp0cdr80wBwqmgKfj5P9p57mPO2fybe0UFLxxJUAZ7vWQdLzxs3nW3bnmUJ0DpvyR5fv+HAJLFYP2A4JZpCqIECeAODVfd1B7Q2m68gFHzl85uNv+Hs5mOAh7FSB7amYAA8z2Pp0qUcfvjhAFx++eX86U+jPeAvueSS6PX+++/fL3OcSTyz9lEaJYft6yiQYWe46lhxPVTMouWSS1DFIic/OsLW7FZGHK2Ob0hvoCYQCm42E63vGekh7+VZ3LAY5fs4gYrv9vZS2l/E3aG1i8SyQ/XrnDkAbO2q7Gju2a41iOSc9orbDbMXu7ERu6WF4qbRiCBny+j7icxHocAo5LPjtj2+83F25nZySutxAEhy6oXCAV0ldVfszhP9/qLUcTfbnXiToW/9A4AOx0w4Wig0JBoqjhXXx7ctUocfTu2qVRzy+2dgNWwc2shRc45iw+AGjikIoEgVtcnoiNYj2JzRP9ZFjYtwd+5EFYvED1mMs2kzfjaL3aDP52zvBiDeqaNG7BbtQO7ZvmHcXAAGd+rjxlpa9sn/wnBgkVi8mOLmUUEQagqSSk0sFPq1puDkRsZt+9XGX5G0kxzdcDi9HPg+BQNg2zYbN26M/AXf+973OPPMM6Ptt912W/R66qmn7pc5ziRi3Y+SlRotFFzIFCvbWQEsz0fF9Fe47owziG3vI+6oKAJpQ3oDjY5+7qkpKrZktGofvi5uWByp+7XH6aoqpRFITreOcwhDCe2WZgByvd3k3FzZXJRSjPR2B+OMUDgYiR+ymOLmUZ+Cs2ULdmsr8Xnz8AarC4XQ3+AUyoWC53v8dtNvOWPBGSSCnDWjKcwCUqkU3/nOd3jjG9+I67qceOKJvOMd74i2DwwMcOyxx5JMJrnlllvG7d/d3c2qVasYGhrCsiy+9KUv8fTTT9PY2DidlzEtFFyPzuGn2dp4JDFvBwl3V+YjH1WjI7rspiYAmgp2FIFU6lNIOZQJhZjEmFc3j+zmBwGoOf440nfcgdvTQ3KZDm91u/VNPtaho4liQahp/YgWPMvnjGqiffk+Ehl9MiMUDk4SixYz9PO78ItFrESCYtcW4osWIpZd1dHs5/OoIIzVHVOapyvbRW+ul5cvfDnqCb3NSh3AjmYDfPSjH43eP/bYYxXHvP/97+czn/lM1WPMmzePrq6uqttnE09t7mE5m3h23luIeTtQsitNwcO1taZgN2uhcKjVwYb0BoadYbqHu0nmtI+g0U2wKTAbbR7azIKGBcSsGMWuLWDb1Bx7LABu76iz2enuxm5uxqqp0ecIbvaNI/D8wPNlQmFjeiMNwYNeKKAMBxeJQxaDUjhdXSSXLcPZ0kXNypX4IyM427ZV3Cc0HQH4TgHHd4hbOtJw54hO4ZpXNw8/p/1VEnwXpxJjPjLMGDY9/SBJcak/9ARi/qhPoRqWq1Ax/VwT3ogPs+fxwuALvJh+kbirsFxds6DZT7FlaFRTCMNRnS1dxDs7IxORVxKW6m7vJtbZGS3bTU0gQkve4vnB8vDhTUObaMgpaKhHYuZZ62AksXg0LFU5Ds727cQXLcRuaa7qU3D7R9cnXEgXRnNzenP6u9he044qBJrCgRySatg1Gzdu3N9TmFJufeJ+Gq3F1UNKgcHiTmpjjSSsFL3Paidzzfzl5AFbQSY3WHVfy/NRcV2IzArMaYvVHLZknmR9//rIdATQ6MXZnNmMUorNmc0cN1dHcxS3bCa+aCFWUxMSj5eFpTrd3cTnz4+WJRbDbmxkgWvzwOBzZXPZNLSJprxFvGX2ZjMbJiZ+yCGA9iU427eD55FYuAgcB29goGKJEG9gVFOIBUKhraYNKBEKte0U8/rLLNMQkmqEgmFKuHfjOj75+NsZ2fyPeMNHVB1Xd9gncQZOodj7Kj4XX0M22UIxPuovyWerZzVbngJbCwW7qRmA+aoRT3ncs+UeGoo2oDWFOseme7ibHSM7GHaGWdyon+qcLV00vOpViAh2e9sYR3M3tSeUt/WwW1vpcIo8P1CuKWwc2sjZxaSJPDqIsZubserrKW7aHGU2xxctxBscQBWLqJERZEzZjFLzUdyDwcJgtNyT6yFuxWlMNNJTyCOJBGJNvXHHCAXDlLAlrW+u7zxqJ2+af0jZNlEuiYFnie18jDOtDJen7uQ/629BvDz+krNxnNHInpHhwarnsD2Figfmo8Cn0O5qm+tftv2F02KdwCYklSJVVCgU92/TuSCLGhbhZYfx+vuJB4lHsfb2SFPwR0bw02liHeVFzOyWFlpyPewY2c5QcYjGhBZgm4Y20Zy3sRcYoXCwIiJRWGqYEJlYtGg0OXJgkMQYoRCZj5oaiLtZBvOD0bbekV7aatoQEfxcflq0BDBCwTBF5Pt0BNCyF29i0ZrrK47paz8c6sGZeyTWYSsAsI+5GCc7KhTyI9UdzbanIBaYj+rqwLZpKtjQAEW/yBJrLrCJeEcHfk4nBt237T5ACwWnSz/NJRYFQqGtPQpRdbp14lqYoxCds7WFuhf0tg2DG1g5dyWe77E5s5n6EdtEHh3kxA9ZTP7pp3G6tiDxOLG5c6PvhDcwAAvLS6B4/f0Qj2O1thL3suM0hfYanQipCvlp8SeAEQqGKSI3rJ+4B464EE5425itAq1LyRQG4I6/wZm/Al72yWiru/YPhC1L8lWKhEEoFIKaMCLYTU1YmWEWLFnA1uxWFlk6AznW0YHzjFbT7992P5ZYLKhfQH6LLnMcXxhqCm3kHn1UzyHIUYiNKXcca2khPqSF1nODz7Fy7kq2Zbfh+i7JjGeEwkFOYvEhZH77OwobNxJfsACx7Si/pVKugjvQT6y5GStVQ9wpNx/15npZ3KDNnH6+MC2RR2Cij/Y5+7p0NsBZZ53FEUccER1n586d48bceOONiAi/+93vonV33HEHIsLtt9++x9ezp+Tz2hfgNXbC/OPG/K2EVBNDRX3DH1uK2i2MagrF3PjU/xDbU0h89LnGbmzES6dZ2qTzDDrRpp1YRwcql6M+VsdQcYjOuk4SdgJnc6ApLB7VFLyBgSByJMxm7iw9JXZLKyo9RK1dw5qeNYD2JyQchVV0ohuA4eAksXgxuC65hx4mHmqgpZrCGLz+AezWVuxEkoQn44RCe22gKeRzRlM4UAlLZ4POU6ivr98nZa9vvvlmVq1aNeGYY445hltvvZWzzz4bgFtuuYUVK1bs9bn3hEJR38zD6KBKhDkI44VCnvDr74xMJBSA2Gj1WLupCT89xKFNL+XerffS7tWhgPi8DnBdltQuZO3QehY2LNTn7dqC1dSEHUQuxdp01Ifb14ezozxxLTpHSwu4Lue2n8FPn/8p7bXtNCeboxwF42g+uAkfMLx0OjJL2hMJhYEB7NYW8Hxq/FgkFBzPYbAwGEUi+fnCtPkUjKYwDexN6ezd4YwzzuDBBx/EcRyy2SzPP/88K1eujLY/8sgjnHnmmZxwwgmce+65bN+uTSTf/OY3OfHEE1mxYgUXXXQRI0GG5erVq7nqqqs47bTTWLZs2W5pHMVAKLgTREuEQqHgF8rWu8XRZSdXOU/BVz4xnzJNwWpuwkunecXiV3By58k0uzrbOdaun7aWJXR4aaiSO1u6oh8uQGyuHuf29OJu78aeMwdrTA+MWKv+gf/H4e/kDYe9gW88+Q3+5/H/YZ6r+28a89HBTXzxaFBFqClYDQ1gV85q9vr7ibW0IokESd+OhEIYjhoKBZU3PoV9wy8/AN1r9u0x5x0D53161+MC9lXp7H/8x3/Etm0uuugirrnmmorF80SEs88+m1//+tek02nOP/98XnzxRQAcx+Hd7343P/vZz2hvb+e2227jQx/6EN/+9rd5wxvewNvepu3+11xzDTfccAPvfve7Adi+fTv33nsv69at4/zzz+fiiy+e1HU7zggkoTBBjb/QfOR4Ttn6UvORW6FImD5+QTeNj5drCsUXNnBCxwl865xv0f3wp7Dq6rDqdYG7Q2JzgVGhUNyymdRRR0X7R5pCb4/OUajQPjG86cvgEB899aMsrF/IdY9dx0tYAmSwm5urX7Bh1hOb246kUqh8nsQirZGKZWE3N1csn+0OaPORn8+T9K0oea0np31yoaPZz+en7btlNIUpZl+Uzr755ptZs2YNf/7zn/nzn//M9773varne/Ob38ytt97KrbfeGh0bYP369axdu5ZXv/rVrFy5kk984hNR+Yy1a9dyxhlncMwxx3DzzTfz1FNPRftdeOGFWJbFUUcdxY4dO8adrxqup2/mE3lRqvkUvOJoDRgvX154LsQp6vVWmfmoGS89mtfgZ7JYjY1YtfopfmHgeF7UuAjleThbt+nkooBQo3B7enC7txPrrCQUdHKaNzCIiPC2Y9/Gt875Fpd0vj7YbjSFg5kwLBVGNQWgYlazchz8oSHs1hYkkSDh6sZQUKIp1JZoCtNQ9whmu6awG0/0+4uxpbM9z+OEE04A4Pzzz+faa69lwQIdxtbQ0MBb3vIWHnzwQS677LKKxzvppJNYs2YNtbW1kSACXcVz+fLlFQXP6tWrueOOO1ixYgU33ngj99xzT7QtWaKylvYa2BWer2/aRd+pOiYSCv5YoTBqPqoqFIK0/zJNobERP5NBeR5i23iZIez6+qjP7vLaQzm2/VhWtK/Qxe5cN8pRALCDfgluby/O9m5qTz5l3HlH7cOjSUcnd55Mv/scOzBCwQDxxYsoPPss8QULo3VaUygXCqE5KdbaipNMEPdGy1yUlrgA8AuFaamQCkZTmHJ2t3S2bdtRT+drr70W13XpDerxOI7DXXfdxdFHHz3hOT/96U/zqU99qmzdEUccQU9PTyQUHMeJNIJMJkNnZyeO43DzzTfvk+v2Az/B2Bt+KVUdzSVCwSq648xLUKIpjDEfAXhBC0Q/k8VqaMCq05rCHOq4+bU301bThtMdRheNlrGwEgltgtq4ET+b1Q7qMYQ+hbE/cG9gAEQip7Xh4KXupJNIHXMMdv1oolqspWVcSGr4HbIDn0LMVaSLaXzl05PrQRBaU1ozVfn8tPRSgNmuKcwA9rZ0dqFQ4Nxzz8VxHDzP4+yzz47s/9U477zxrSITiQS33347V111Fel0Gtd1ufrqq1m+fDkf//jHOfnkk2lvb+fkk08mk6meMDZZfKVv7AWvUHXMUCHwKYzRJvwSoZBwIOtkabHLn8DdsEBYfNQRHGY1++k0tLTgZzLY7W2RpuAPjzqtR3+QzWXHjc1tJ79mrX4/rzwcFXSVSkkmywqZgY5Bt5uaELt6tJXh4KD1sstoHaPJ280tuAOPl60LS1zEWluQeALbVfjKJ1PM0DPSQ0uqhZilb9F+Pj9tmoIRClPIviidXVdXxyOPPLLLc61evZrVq1ePW3/jjTdG71euXFnmzwi54ooruOKKKybcFyCbrR4eOhZf6Rt9paf8kGqagueMLidcyBaztKTKhYJTDM1HJUIh1BQCv4KXzZJYujTyKfgjo05rb3AQGB9Care1MXK/Lsw3NpsZgiS51taKpgBjOjJUw25pGVcUzw2Egt3SgiSTWI4L6AS23lxvZDqC6fUpGPORYZ/jej5KtDDYE/ORKpYKBUXWGS+MwggluyRkNKyUGgoFP5PBaqgfFQolmkJozx0b0RE6m4GK0UcQOA1LCpmBFjJGKBiqYbe0gOfhl2jhXqBt2q3afCSOLt4YCoXIyez7qGIRSZmM5lnPxo0baQvCIGcT2YILop96JjQfVXE0+yVCIe5SUSg4TmA+ipVqCs0AeOkhlFJ4Qb/lyHw0RlOQRGJc6YBYWyAURIjNnVtx3rGWVtxx9mEjFAzViUpdlGiY3kC/9kM1NSGJOOL7WL4iXUiX1z3KT1/XNTBCwTAFDOVcEP3UMxnz0dgxfgXz0Vi8IAGwNLks9Cl46TSqUADHwWpoRJJJsKxxQsFuaRmX7xFqCrG2trLIplLslpboKS863sBAdH6DYSyVSl24/f3Yzc2IbSPB9zjuQl+uj/5c/2g2c/BdN9FHhgOWdM7BD4RCNU1BKVVVU1BOICSSSS0UKpmPAk0hFh99erIj89FgpKbbDfWICFZd3RhH82DFZKAwgS3WOd7JHJ2ntaXsx62UwhsYMCUuDFUJtcjSrOaw7hGMdlSLeboMu6vcsmxmADGaguFAZSjv4IsPVPcp5NwcnvKoi9fhKx/Xd6NtofnIaqiPoo/GEoat2iV5FBKLYdXX46XTeIFQCLOZrdraiprCWGLt+odYzZ8A+qnPz2ajeaqREVSxaMxHhqqM5rcMRut0iYsgQz7QFFK+HbV6jXIUcqH5yPgUDAcomeFh4jnFjZ93mbupcunrUEuYk9IJY6XO5lBTsBsaqpuPnEAoxMtrE9mNjfjpdKQpWA31+rW2dlxIakVNITAfVYo8is5RktUMunkK6LBDg6ESlYriuYOjmkIYRddqNYwKhdrRXgpgNIUDmh07dvCWt7yFZcuWccIJJ3Dqqafy05/+dJ+fZ926dZx66qkkk0k+97nP7fPj7ym5bJpUVqgtQsvOyhnJoVAIVeSyXIVQKNQ3kHKloqYQZj3biXI7qy6KN4SX0fvYDYGmUFdXQVNoHnfcWEcHkkySWLqs6vWNzWoezXkwQsFQGauuDuLxsgQ2bT4KNIVA4221Gtia3QpAWyqskBpqCiZP4YBEKcWFF17I5Zdfzg9+8AMANm3axJ133rnPz9Xa2sp1113HHXfcsc+PvTfksmmUFzhwi5UdzaGTeU7NeE0Bx8W1tZ21JmNX1hQioTBGU2jSlVL97HjzkRrWQkH5Pl46XVFTsOvrWfaLXxDvqBx5BOOzmsOcB9NLwVANESHW3Bz5FJTv4w0OEgs1hYQOamix6qN9opDUMKjClM4+MPn9739PIpEoy1o+5JBDoqqjnufx/ve/nxNPPJFjjz2W//3f/wXgnnvu4ayzzuLiiy/myCOP5NJLL91lraG5c+dy4oknEq8SJbO/KA4PQSAUrKJbcUyYzRyZj0p8D8p1cG1BUimSrlVZUwgilGKJcjtrWBTPK3E0gxYK3og2H/lDQ+D7VR3DiYULqkYeQYnTMMhVCJ/+jKPZMBF2S0v0AOGl0+D7kSky9Ck0Wzp8uj5eT01Mf7f9nNa2TY/mfcBnHvwM6/rX7dNjHtl6JP9x0n9U3f7UU09x/PHHV91+ww030NTUxEMPPUShUOD000/nnHPOAXTW81NPPcX8+fM5/fTTue+++3jZy162T+c/Hbi5NBIIBakiFDKOvmmH5qNSTUEcDz8mSCpJsor5KHTyxuPldtZIUwjMR2FCm1VXF2kK1RLXJktoBw59CsZ8ZJgMOqt5EBgtcRGaj8LQ6iZLJ1qGvwsY1RRkmvopTJmmICLfFpGdIrK2ZN1HRWSriDwe/L22ZNt/isjzIrJeRM6dqnlNN1deeSUrVqzgxBNPBOA3v/kNN910EytXruTkk0+mr6+P5557DtAVThcuXIhlWaxcuZKNGzfux5nvOW5uKDIfWY5XUeMJNYVKQgHHxbMtrGSKZBVHc5jLYI+J3bYbG/GGhrT5SCTKZi7VFCJzz54KhaYmEIl+2O7AAFiWbqZiMFQhLHUBpXWPAk0huOE3oL/PoZMZSnwK09SjeSo1hRuBrwI3jVn/RaVUmVdURI4C3gwsB+YDvxORw5VS3t5MYKIn+qli+fLl/PjHP46Wv/a1r9Hb2xu10lRK8ZWvfIVzzy2Xe/fcc09ZmWrbtnHdyk/ZMx1VyGAFU0+4Ctd3idvl5pjQpxBWgSxzNLsuXsxCUkniVcpchEIhPsbRbDc3gePg7NiBVV+PBJ3fSjWF8GltT5/sxba1RhKYjcJIJpmgy5zBEPZU8DIZBn70I70uKNcemo8a0Df+0MkMJXkKB7qmoJT6E9C/y4GaC4BblVIFpdSLwPPASVM1t6nkla98Jfl8nuuvvz5aN1IS9XLuuedy/fXX4wQRNs8++yzDJaGSswFVyEY+hYRbOYFtqDhEXbyOVEzf1MvNRy5+zMJKJom5imFn/P/HD/5/sbFCISiK52zpisJRIQhJzeUiBx/suaYAEJs7l8Hbf8zWf/938k8/Y0xHhl0SC3wKL7z2tQzd9QtaV68medhhwKhQaAw0hdDJDAdH9NG7ROQy4GHg35RSA8AC4IGSMV3BugMOEeGOO+7gve99L5/97Gdpb2+nrq4uqoT6z//8z2zcuJHjjz8epRTt7e27jB768Ic/zKpVqzj//PPL1nd3d7Nq1SqGhoawLIsvfelLPP300zTu55r+UqIpxN3KCWxDxSEaE40kbP1jKB1juT5+zEKSKWJFL9IqSol8CmMczVYoFLq6sOtHzTlWXS0ohcrlSqKF9vxGvuBLX2Lg+98jfefP8bNZagNN0GCoRmxuByhFvGMei67/OjVHL4+2hXkKtSoOwpgKqdMbfTTdQuF64OOACl4/D7x1dw4gIm8H3g6wOGh7N9Po7Ozk1ltvrbjNsiw+9alPjWuCc9ZZZ3HWWWdFy1/96lej99dee23FY82bNy9qqTmTsNwMtieAIu6Nr4IK2nzUkGggYQVCoVRTcD0tFFJJbMdjuJgtKzkMOkIJIDEmy9NuDITCjh3UlJSqKC2K5w0MQCwWrdsTksuWMu/DH2bu+95H5ne/I7FkyR4fy3Bw0HThBSQWL6L25JPH9d2wkvp3UKf0a6mj2c/nwLJgmqIMp9UIqpTaoZTylFI+8E1GTURbgUUlQxcG6yod4xtKqVVKqVXtJWWODTMHcbMkXO1cTjiVhcJQcUgLhUBTKC2KJ66Hsm2sZApRejnnlifBhVnPsXgFnwKA52HXl5uPQJfPDhPXxhbD2xOs2lqazj+fmmOP3etjGWY3VipF3WmnVWzEFJqP5iXm8DfL/oZT558abVP5ApJK7ZPv66TmOS1nCRCR0ipjfwuEkUl3Am8WkaSILAUOAx6czrkZ9g1F18dWw8RDR7NX2aeQKWZoTDRGDuix5iMVs6O0/oTLOL+CChqSxMZGHzWNViq1SsxoZZrC4ACxvfAnGAz7mqhKqgefOuNT5ZpCIR8VzJsOJm0+EpFapdTIrkdG428BzgLaRKQL+AhwloisRJuPNgL/AqCUekpEfgg8DbjAlXsbeWTYPwzlHZJWnsQkfApHJo6saD6yPB+VtCMbasLVeQ3tlNhZXQcfYMxTV6lQsBsqaAojI0GFVOMYNswcwsgivzD+t6JyeWSa+jPDJISCiJwGfAuoBxaLyArgX5RS75xoP6XUJRVW3zDB+E8Cn9zVfAwzm6GcQ0JykVBIuNV9ChM5mlUsFtWPTzgwXByrKTh4NuNUakmlkEQCVSxGJS6g3HzkDg6QnKC2kcEw3YSaQmnXwRCtKUyfUJiM+eiLwLlAH4BS6gng5VM5KcOBy1DeJVGmKahxQsH1XYadYS0UKmgKtuuj4nbUaSrUFMoP4uLZ422sIoLVFGQxl2oKZeajynWPDIb9hVgWxGIVhULoU5guJuVTUEptGbPKmHYMFUnnHOKSj3wK8Qp5CmGGcmOyRFMoFQqegniJplDBp6CFQuU5hCYku2G8puBls6afsmFGIolEVNKiFFXIT1s4KkxOKGwJTEhKROIi8j7gmSme1wHNdJXOvvHGGxERfve730Xr7rjjDkSE22+/fZ+fbzIM5RxsKZaZj8a22wzzDhoSDaOO5jJNQemQ0RJNYVypC6eypgCjvZormY/cnTvBdY2mYJhxWIkEyqlgPsrlp62XAkxOKLwDuBKdTLYVWBksGyoQls5++ctfzoYNG3jkkUe49dZbpyyf4JhjjinLibjllltYsWLFlJxrMgzlHWJWcTT6qIKmEPZSaEw0EpMYgkQ+BaUUtqeQeDxSmSuWunA9/KpCIdQUxjuana3b9DajKRhmGJJIREmZpcwon4KI2MCXlVKXKqU6lFJzlVJ/r5Tqm6b5HXBMZ+lsgDPOOIMHH3wQx3HIZrM8//zzrFy5Mtr+yCOPcOaZZ3LCCSdw7rnnsn37dgC++c1vcuKJJ7JixQouuuiiqBTH6tWrueqqqzjttNNYtmzZbmscQzkXS4pRnkKl6KNQKDQkGhAREnYi0iY85RHz0OajxASagquL5lUi7NVsNYyGpEoigcTjONsCodDcVHFfg2F/EQZIjEXlCzMn+kgp5YnIISKSUEpVbrY7g+n+1KcoPLNvS2cnX3ok8z74warbp7t0tohw9tln8+tf/5p0Os3555/Piy++CIDjOLz73e/mZz/7Ge3t7dx222186EMf4tvf/jZveMMbeNvb3gbANddcww033BAJru3bt3Pvvfeybt06zj//fC6++OJJ/38ywzmwvDJNYayjuVQoFF58kUYnHgkOx3eI+TrtPzQfNfjJcZqCuP5uaQqgnc3OVp0TaXofGGYakkyiKoSk+vnctGoKk8lT2ADcJyJ3ApG3Tyn1hSmb1Sziyiuv5N577yWRSPDQQw/xm9/8hieffDJ6Ak+n0zz33HMkEomodDYQlc6eTD+FN7/5zVx33XWk02k+//nPRyU01q9fz9q1a3n1q18NaC2lMyj9sHbtWq655hoGBwfJZrNlVVsvvPBCLMviqKOOYseOHbt1vYWRNEWR8jyFMUIh9Ck0JhrZdMnf8toVRUaOLBEKHmXmowZVSSjoonmVCLWAsaWsrdpanEBTMj4Fw0xjQk1hGn0KkxEKLwR/FnBAFYyf6Il+qtgfpbNPOukk1qxZQ21tLYcffni0XinF8uXLuf/++8fts3r1au644w5WrFjBjTfeyD333BNtK53HZExYpRRH0hREaC7JaC6O8SmEQqHeTzAwOEhjro7BQHAUvWIkFMIsznqVoG+c+cjDr2I+qjnueGpWrIhq1YdYdbUQ/E+NT8Ew05BEvIpQyGOlpqeXAkzC0ayU+phS6mPo4nWfL1k2VGB/lc7+9Kc/Pa7I3hFHHEFPT08kFBzH4amnngIgk8nQ2dmJ4zjcfPPNe33+EC+XoVCiKQA4+fJE+KHiEDGJkRjWP4CUO+podtwiMR+sxKimUO/Hx2kKOsGt8te37pSTWXLbrVFCULRPbVAAzzTEMcxArPh4TUEphZ+f3uijyWQ0Hw18D2gNlnuBy5RST03x3A5IprN0dinnnXfeuHWJRILbb7+dq666inQ6jeu6XH311SxfvpyPf/zjnHzyybS3t3PyySeTyYwvT70nePkhiiLapyACSuHl8mVjwgqpflr7FlKORI5mpxD0o40nI02h1ouNNx95flVNoRpWnY5AspuaTEMcw4xDkkm87JjfoeOA709rnsJkzEffAP5VKfUHABE5C13h9LSpm9aBzXSVzl69ejWrV68et/7GG2+M3q9cuZI//elP48ZcccUVXHHFFRPuC5DNju96NiH5DMWkdjBbDfX4QxncwhhNoTBEY7Ix6muQKIlQKgZjrURClwq2LJKeUHDLTVCW66F2s0hYmNVsTEeGmYj2KZTn9PhRf+YZEpIaUBcKBACl1D3AnheiN8xuihkKCHFvtLeBlysvez3kDNEQb8BLDwLlEUqhpmAlEogIkkqRcq1xuQ7i+ajd1RSCXAXjZDbMRCo5mlXUn3lmCYUNIvJfIrIk+LsGHZFkMJShlMIuZnGDVpxhvoBfqGw+CjWFpDNaH8kNhIId11qAlUyScCHvlR/DchUqvns9oiQUCkZTMMxAJDm+zIUf9WeeWULhrUA78BPgx0Abu9ktzXBwkHd8UiqH62uhEBam88d80SPzUToNQLyocHytNruBALED05CkUronw1jzkeePK5u9K+zQfGQS1wwzkAk1hZnkaA56KF81DXPZZ4xt3WiYOkpDVofyDvXk8CJNQd98w6edkFFHcyAUHH/UfFTMkaJcU4g74zUF21Oo+O4JhVBTMIlrhpmIVUEo+EF/5hlVJVVEfisizSXLLSLy6ymd1V6QSqXo6+vb7fh6w+6jlKKvr49U8IUdyjnUSQ5P6WeN0HxUmqWplGKoOERjohE3MB/Fi37kaI40hcSophB3FXk3X/aZWp7abU3B+BQMMxmpFJKaD3xsMyz6qE0pNRguKKUGRGTu1E1p71i4cCFdXV309PTs76nMKrKFPJliFvFHS0coYHvW5Qdrh8k6Gyk4Pu8mT6+vK59ajToXQBVHTT8Fr4DjO4GmoF1TsRJNwS1qoRC22ZRkgriTQ6FNTGGpbdvzYTd9Cib6yDCTkWRyXEG8SFOYYWUufBFZrJTaDCAih6DvBzOSeDzO0qVL9/c0Zh2X/OhjrB25nTMSXyMhpYlfMQ6bN2qjP7pbeDFjA15kPqLEpxD2RaiP1+MNDOojFL0oTyESCgn9I7CSKWKZAUCbkEaFAhDbTaFgNAXDDEYSCXBdlO9HeTSqMP3RR5P5VX0IuFdE/ggIcAbw9imdlWHGMVTQN/N/O28BhzYfWn3gD2L8PB0IhcDRTEnsdSgU6uJ1kaPZdn2cwJHsBVpFpCmkktj9uqdTwS1AkKSsy2vvnlAI+yzE2tomHmgw7AdKW3KGPgQ/N/3RR5NxNP9KRI4HTkFrCFcrpXqnfGaGGUW2OAw29Of7OZQJhEIhg/L1U07oU6AwKhTCzOS6eF0UkqrHaLU5jFSKJ3WtFyuZwna0UAidzUopbJ/d1hTqTj2FhV/9Cqljj92t/QyG6cBKBkKhUIBAKESawkxoshOUzG4CCITAMHAOcJmIJKrtZ5idjLg607g/3z/xwEIGFUQfWYH5SCpoCvUxrSmEEUGRUAg6T9mRppDCKupCSmFYathzQeLx3boGsW0azj7bRKYZZiSlmkJI5FOomRkF8X5IkLksIiuBHwGbgRXA/0z5zAwzirynoyAmIxTCDt6h+UicCuYjX1eEjHd06LFFF1/5eMGTUTwRaAqpJFYhEApBVrPjBT0XYrsnFAyGmUwloRBpCrtZ0mVvmEj/rlFKbQve/z3wbaXU50XEAh6f8pkZZgx5x8NVBWLAQH5g4sHFLLhaGNhBJVIpjpZMDc1HNSMeBSA2r4Piiy+SdHQvhVBTCM1HkkxhBZpGztWCySmGRfOMUDDMHkKhUBqBFPkUZkieQqmO/UrgbgCllD+lMzLMOHoyBcTST+mT0hSCVpxSW4tvS2T+ARhxtBmqdkSrE/GOeQAkHF3/aFQoaLOSlUpGjupIUwgilIxQMMwmwvazYzUFicentarvRJrC70Xkh8B2oAX4PYCIdAIHXGtOw56zM1MA0R/5hELBc8HNI4FQsJJJvHgschTDqKYQzwahp52BUAiK4nnBDyIWJq8lU9onoezI0RwVzTNCwTCLkIT+Po/1KUynPwEm1hSuRtc72gi8TCkVGobnocNUDQcJPZn85DSFoKOaFWoKqRQqYWOVCoViFkssYoFQCDWFsCieCvwPVqBKh81F4u6oozk0HxmhYJhNVPQp5HPT6k+ACTQFpWsKjGsKoJR6bEpnZJhx7BgqgKVv1hMKhUIgFBxtYZREAj8eI+4UcH2XmBVjxB2hLl6HP6hzFGLztKM5GfRUUIH5KDQNhQ3LE+6o+SgshWHFp/fHYjBMJeHNv7RSqp8vTKs/ASZXJdVwkLOzRFOY0NFc0KYhy/Xx4jYigkrGy/olZIvZssS1+Lxyn0LYZCR8apLgh1JaPntUKBhNwTB7qORo1v2Zp1co7F72j+GgpDs9glguMWCwMIj73QuIVYr1LwzhAnFH4Yf20UScuKdv+LXxWoadYV3iIp1GUqmo5ETSCbqvBeajSFMIzEcJB/JuIBQcLaCshNEUDLOHynkK+ZmtKQQVUk066EHGjqzupdzp6CiiQScDxeHxf2JTXHomcQ9UInjeSMS1FhBUQR12hqNsZrupqUwTcDwH5bj4QlQBVUrMR+M1BZNDaZg9jAqF0bwelc/PHJ9CiIjcA5wfjH0E2Cki9yml/nUX+30beD2wUyl1dLCuFbgNWIJ2YL8pqLoqwJeB1wIjwGql1KN7eE2GfczObAaaYIGVZAse/Rd9nbaWwyuOLeYHSdx5GirQFEgkiOdV5A8YdoapT2hNwW5qwgoiK5KB+QjHxbMlyjoOHc21nh05ml0nTwywjVAwzCKikNRSn0KhgN00vU2hJqMpNCmlhoA3ADcppU4Gzp7EfjcCrxmz7gPA3Uqpw9B5Dx8I1p8HHBb8vR24fhLHN0wTvSNaU1iYaAYmdjYX/SIJl0goSDJJPNACQIekRppCc/OopuAorU24Dp49apoK7al1fiISLFHRvMT0qtUGw1QShaQ6Y30K06spTEYoxILchDcBd032wEqpPwFj7x4XAN8N3n8XuLBk/U1K8wDQHJzTsJ9xPZ9sPhAKdfMB6M9VFwoFr0DcBQJVmESiLHJo1KcQmI9EUKmkjj4KNAW/RCiEQqPOj0Xmo1AohGGrBsNswKrmU5jGCqkwOaFwLfBr4Hml1EMisgx4bg/P16GU2h687wY6gvcLgC0l47qCdYb9TG+2SLu1E4AFzcsAGChUj0AqelpTIKj4aCW1UBjvU0hHTmZJJSO/g7geXmz0azmqKcRHC+KNSXAzGGYD4QOQGht9NI29FGASQkEp9SOl1LFKqXcGyxuUUhft7YmDPIjdbtYjIm8XkYdF5GHTXW3q2ZnJM9feAcC8tqOwxKIv11d1vNYUFBIKhVQqCklVSmmhEKvVPoXmoIpqUmsKjudoTaFEKIRPSbWlmoJjzEeG2UcUkjrGpzDjNAUR+ayINIpIXETuFpEeEfn7PTzfjtAsFLzuDNZvBRaVjFsYrBuHUuobSqlVSqlV7e3tezgNw2TZOVSg3dLtM+rnHE5LsmVSmkL41GMlU8QDoZBzdVvNRi8JjlOiKaSiPAVxPXy7VFPQx6nx7SgkNRQKttEUDLMIsW2w7fLoo1xuRvoUzgkcza9HRwy9BHj/Hp7vTuDy4P3lwM9K1l8mmlOAdImZybAf2Zkp0GxrzaC2oZOWVMuEPoVQKFhjhELBK0R1jxr0vT2KqrBqUlGeguV4+PESTSEwH9V4duSX8INcBqMpGGYbkkhE5iPl+7oL2zRrCpNJXgvHvA74kVIqPZkmJSJyC3AW0CYiXcBHgE8DPxSRfwI2oZ3XAP+HDkd9Hh2S+o+7cQ2GKWRnJk+DpbOPa+N1zEnNmTD6qOAVtFAIbuZ2qibyKYwVClYkFGpJDEDeK5L0PPyYHR0vFC4p14rMR37gaI4boWCYZViJRBSSGr5Ot09hMkLhLhFZB+SAK0SkHcjvaiel1CVVNr2qwlgFXDmJuRimmZ2ZAm12BkhQG6+lJdXCM/3PVB1f9IrUuKM1i2KpGnwfisVcVDa7bkS7kmKB+chOpUg4iozvYLk+KjYaVRSaoZKejDqandDRbISCYXYhiUQUkurnp78/M0zO0fwB4DRgVVApdQQdQmo4CBgcTBOzctgIcStOa6p1lyGpCVdrCKBv+ABOPhdpCqkRnRkdagp2TW0UkqqFwqimILEYxGKkXCsyH4WVVMNGPAbDbKHMfDSiH6JmXPSRiNQC72Q0oWw+sGoqJ2WYOcTSLzIiFrV2AhGhNdVKxslEBe7GEuYp2EGmcqymDgAnN8xwUbfiTA3rm3roaLZSNSQdqSgUQJuQkt5o57XQpxCPG03BMLuQRCIqiOd0dwMQC8rLTxeTcTR/B91U57RgeSvwiSmbkWFGUZ/dyIgl1MR0J7SWVAtQvVqq4xZIeBBL6fHx4NXP5xh2tVBIDAfRQ2H0UU0qKp1tewri5VbNMDppvKZQu68u02CYEUgyiSoEQqGrC4D4wulN2ZqMUDhUKfVZwAFQSo1Q3qrTMEvxfUVrfgs5EWoTut/ynNQcoHqpi2Jeq7yxGn3DDoWDW8iTLWrzUSybR2prowxOK5mKah9Zrg8VNIXSrOhIKJiQVMMso9R8VNzSBSLEF8w8oVAUkRqCRDMRORQoTLyLYTYwMFLkELaTiaWoTdQD0FrTqrdV0RTcnNYGQg0hFA5uboQRVwsMe2ikrMiX1OgEN8d3sD2Fio3XFOKOIu/mUUpFfZxNmQvDbEMS8UgoOF1dxDo6pv17Ppnoo48AvwIWicjNwOnA6qmclGFmsDNTYKnVTTaWoiamfQQtSW0+6stXzmp289ruHw98CVE3qXyBbDFL3IqjhjJlQsFK1eiCeE6BmKdGK6wGSCpJ3CmgUDi+A652VGOa7BhmGVYiiT+sH6yKXVum3XQEkxAKSqnfisijwClos9F7lFK9Uz4zw35nZ6bA0bKdXHwhHYFPIdQUqpmPQqEwmtGsX71CblyF1JAwuqKYyxLzwBlzs7eSKezAyZz38ijHxbVgMvkyBsOBhCQS+INaC3e6tlJ3yinTPofJNtlJAQPAEHCUiLx86qZkmCkM9O5gjmTI2za1cS0UGuINxKxYVfORHwiFMHktzEj2C3lGnJGoFWeZ+SgIXy2MZIh5o13XRrcniRU9PcYtoFwHv9ztYDDMCkKfgl8s4u7YQXzhwmmfw2Sa7HwG+DvgKcAPVivgT1M4L8M+YGcmzw8e/xO4LdTH5uz2/juefpQLgYIItYGmICK0JluragpRwk3gBC5tHJJ1skHZ7O5yTSGo7eKMZCsKBbu+ntg27VzOu3nEcXFtoyUYZh9aKDg4W7eCUiQWzUChgO55cIRSyjiXZxCe5/OuX3yNdk4lbpWHZrqezyObB1i7dYi6wz6OmzmaQvff7vY5LrSegQSMKDfSFECbkKppCl7YKjMVmo+CGvGFYnmF1AqagpMbrigUrPoG7BF93LyXR3kevmWEgmH2IUld5sLp0vVAZ6SmAGwA4piIoxnFH198insHvoHbPYg9clLZNgGO7Gzgmle0cd32YU6r/Qv/2/qH3T6HODl81yLn5SNHM2hnczVNQeX11yT0KYTmI1UoMOx4dPoN4LrYLS3RPqFPwRsZqawpNDZgDWuhUPAKumVnzAgFw+wjNB85Xbq9zEwVCiPA4yJyNyWCQSl11ZTNyrBLuob0Tfktp7XzX2ecO37ASD9D3309X66F7TX1WPPHdkadHE774bjrvhaZj0BrCpszmyuO9wvl9Voi81GxyLBToC2vhUGspOx55HcYGcFW40NNrfoGrJE8EpTP1uW1jVAwzD6sQCgUu7qQRKLsdzJdTEYo3Bn8lbLbzXEM+5Yd2UH9xhoZv3GkH246n8zABqhtZzse/us+hyWTjSsoOVR+ENZ9rcx81JJsqWo+UmGrzJLOawAUHLJOnpZguqVf9tApbQ0HTur4GKHQoHMkaoqBpuC6ePbuX4vBMNORRBK/WMTZ0kV8wQLEmv7v+WSEQrNS6sulK0TkPVM0H8Mk6R3S/YnsZ26GNT8p35jtgdwAQ3/zWXji8xT9Iv35ftpq2nZ53F9t/BXr+9fznuP1RxzWGyrVFObUzGHEHSHn5srMSgAUtENYxkQfSdFh2BmmORNUSG0fnUs4pi4sqR0vz1S2G3Q2dW1B+xTE81BGKBhmIZJIgONQ3LKF+H5wMsPkQlIvr7Bu9T6eh2E3KQ5tACBjW9C0qPxv4Sq49IdkOo6Kxm/LbpvUcX/+ws+5bd1t0XKYhVwTL/cpAAzmB8cfIKjbMhp9FGoKBXJujoasDi0t0xSC4nl1gXEyzG2ItteXCAU3jzh+WctOg2G2EP5eihs3ktgP/gSYQFMQkUuAtwBLRaTUfNQAVK+dbJgWcvkesGGo/Qh4/fcrjslsujt6v214G8e2H7vL427NbCXjZCItIOyBUKopNCebAUgX03TSWba/BK0Ew+gjsSy8mOAGvob6oSKSSmHV10f7hOaj2kAo2GN8CnajFgp1+aA0t1festNgmC2EQkHlcsQXzDChAPwF2A60AZ8vWZ8BnpzKSRl2jeMMgA0Z5VQdM1Qcit5vz+66u6lSim3DWqPoHellUeOiUU2hxEzUmGwEYLAwOP4ggVCQkqd9L24HSW1CTbpArK2tLBt51HykTUv2mOY5o5qCrn+Ucn2U0RQMsxBJjj4Q7S/zUVWhoJTahG6Zeer0TccwWRwvA8CQV70JXigUYlZsUuajvnxf5EPYmduphUKoKcQraAqF9LhjSBWhYDtFwCaRHhkXUTFWU4glxvoU6qPtBa+AeD7KNsXwDLOP0si7/WU+qvq4JSL3Bq8ZERkq+cuIyFC1/QzTg6d00ayhYqbqmEwxgyAsbVrK9uFdawpbs1uj9z0jPcCoo7lGkrgDOuKoKakTzyoJBbvo4cWsMk3Aj8dIhDXs+rPE2sod3mMdzfYYoWCNcTRbroeKmzoXhtmHJEo1hUX7ZQ4T6eCXAiilGpRSjSV/DUqpxmman6ECnq/wRN9BM7sQCvXxehbUL4jMQhPRlemK3u8c0dFNUbnrO3/PC68+B39kZEKhII6LlyhXQP3kqFCw+4fGaQpiWXhxe9SnMKYnbSgU6gsWBbeA5SkTfWSYlYQBGlZTUxR1N91M9Mv6afhGRH48DXMxTJKBkSKupaN88l6+amvMTDFDY7KR+XXzJ+VTCDWFmBWjN6cL4YbmI3lxM342S+G550jaSWpiNRWFQqzo4Y95ilfxGHEX4q5CssPE5o5PyPGTcWoDn0JsrE8hkUCSSRqLdqAp+ON6LhgMs4FQU0hMc2OdUiYSCqUpo8umeiKGydObLVC0vGi51KFcylBxiIZEA/Pr55N1slXHhWzNbmVOag4dtR3szJVrCvTogLP8M+sAaEw0jnM0K6WwHA8/WV6mQiXixF1o1o3XxpmPAFQyHoWkjhUKoLWFesci7+a1pmDMR4ZZSCgU9pfpCCYWCqrKe8N+pmcoR95SxIOPr9rNPlPM0JBooLNOh43uytm8NbOVQ+0ODss3RT6FEXeEhJXA26GFRH69FgrNyWbSxXJNwVUucXd8j2WViJNwFc3aDVIxdV8lE5FPIZYcLxTs+nrqCkLBK2B7PthGKBhmHxI0mNofzXVCJhIKK0LHMnCscTTPHIb6u8laFnNjOipnqFBdU2hMNDK/fj6wa6HQle3i/F8NcunXn498CjknR228FmfnDgAKgabQlGwad96iVyTh6qf+MiJNIcxmnlgoJFK147ZbDQ3UB9FHlqdM1zXDrCRM3EzMRE1BKWWXOJZjxtE8cxjp3cqwJSxIaTNMNWfzWE1hoggk13fpHu5mbneext4RBjOjmkK91OD19oEI+WefRfk+Tcmmceajolck7jK+nWYyQcKDuSN6vV3BfEQqgR3oo/FEzbjNdkMDNUFGs+2BxIymYJh9xBctInHIIdSecMJ+m4MJ4TgAGUlvxhdhYYPWAHblU2hNtZKyUxNqCt3D3XjKo35HFlGQ7Msy4uj6RnNzcVCKmmOPRY2M4GzeTFOyaZyjueAVSLgKkmNyCJJJEg605WJgWcTmVGj4U5LXEE+OFwpWQwM1eV+bj3yjKRhmJ7HWVg799a9IHnbYfpuDEQoHIPlsF5avOO1XvTQNq4qaguM7utZQogERYV7dvAk1ha3ZrdTlFLGMdiy3Dyl6cj2MOCPMHdY+gvqzztTnX7eepkQT6WIapUbdTaH5SMaUqZBkkrgHc4Yt7DmtSAV/gKRGhcLYkFTQlVJTeZ+cmyPmAUZTMBimBCMUDkCcQjeLemDx/63lxGdVRU0hW9ShPo0JbembXz9/Qk1ha3Yr80qqYbenda7CiDtCe0YHotWdfjrYNvl1z9CcbMb13dHoJLSmEHdHeymEWMkEcRdahiHWVrk+fJjABuOb7ADY9Q0kcy7DTuXubAaDYd9ghMIBiCr20jiin9DbcrGKjuZQewiFQmdd54SaQlemi/mDo1HIcwcVvbleRpwRWgIHcXzRIpLLllJ4Zl3FBLZIU0iNyUhO1ZBwoSnrl5XMLh8zsVCwGuqJF31GcpnAp2CEgsEwFRihcCDiD9AUhHe2jcTIOOPNR6H20JDQWZHz6+fTn++PylaMpSvbxUuydQBYrS20DWlNIefmaE57SDKJ3dxM8siXkl+/vmJRvKKvHc3jSl8nkiRcqB9yq3aSslKjfoSKmkKDPp/KZLCqjDEYDHuPEQoHGJ6vEDI0BlabOSNWRU1hrFDYVQTS1uxWFg8liHV2klx2KB1poWekhxF3hIa0Q6yjAxEhdeQRuN3dNOe1Tb9UU0gX0iRciNfUlR3bSqWwFNQOFaqaj8KeClBNU9DXER8Ku7MZoWAwTAX7RSiIyEYRWSMij4vIw8G6VhH5rYg8F7y27Oo4ByP9w0Vi1jBNgfmoabhy9NFY81GYq1Ct3MXWzFY6+n0SixeTWLCAjiGJHM31gwXiHR0AJI88EoCGTX0AZQls24e3k3Chrr78o4sFeQeWqpyjAGDvQiiElVJDs9nYlp0Gg2HfsD81hVcopVYqpVYFyx8A7lZKHQbcHSwbxtCbySNWPtIUGjJuxeijcF1kPqoLEtgqFMbLuTn68n007RwhccghxBfMp2nIoyeznZybo3YwRywQCqlAKKRe1MIlnR8VCt1ZLRRq65rLjm+X+AsqlbgAsAPtwheQCnWNwp4K4XUb85HBMDXMJPPRBcB3g/ffBS7cf1OZuQz095C3R0tG1GUchioUpgu1B/t3fyG/fj3tte3YYlfUFLZlt1GbV8QzORKHLCa+YAGWguzWTSjlk+wfJtYxF4DYnDnE2tuxnt8MlGsKPWktcKwx0UexkgzlSsXwAGI1eoxnS8XtYfe1hsAlYjQFg2Fq2F9CQQG/EZFHROTtwboOpVR4x+oGOirtKCJvF5GHReThnp6e6ZjrjGK4r4uMZdE8om+esaJHMVvZfBTDpu8jH6f3f64nZsXoqO2oqCl0ZbqicNT4Yi0UAKzuXhpyYLke8Y550fjkkUdSXP8stbHaMkdz76D++Kwx0UfhDR+qm49CP4QXqywUQp9CqCkYn4LBMDXsL6HwMqXU8cB5wJUi8vLSjUpnRFUswqeU+oZSapVSalV7lRvMbKYwsI2sZdGUEwjMLLGBLL7yy8ZlihnmO/WofJ78mjUAzKubx47hHeOO2ZXtYt6A/ndr85EWCu1paA0sU6H5CLQJqfDCC7TajWWO5sEhfWwZ0ySnuWH0c6pmPorVBuajKn0Swp7OoU9hbCMeg8Gwb9gvQkEptTV43Ynu23ASsENEOgGC1537Y24zHS+ti+E1DPskly4FqJjVPFQcYuGINuM427bh9vfTUdvBjpHxQmFrdiuL0lrAJBYtIt7RgRKhPa2YMxTkKMwbFQrJw14CrsuSbCqKfPJ8j/SQ1tzG5inEgnBTq76+LMqolHidvul7VXov25FQCJaN+chgmBKmXSiISJ2INITvgXOAtcCdwOXBsMuBn0333A4EZHgHOU9IFhXJI44AdPXRSkKhc3jUYZtfs4aOug52DO8oK00BOvJo6VCKWEcHVk0Nkkig2lq0phD2QCjRFOILdQXHhZl4ZD7qy/dhObrHw9g8hTDDuZrpCCAWOpqrCAWJx/GTiVGfQsIIBYNhKtgfmkIHcK+IPAE8CPxCKfUr4NPAq0XkOeDsYNkwhvjITvyCvtknDz8cgOYKYamZYoaOodH6QLkn19BR20HRL46rbrpteBvzBrXpKMSe3xlpCsqSMrNPWOu9Iy2Ro7l7uJt40HJzbJkLCQrkVTMdwWiEUjXzEYCqrxk1H8WN+chgmAqmvaehUmoDsKLC+j7gVdM9nwONmkIvWpb7JA9dhrIsmiuYjzLFDHPSPlJbS2LBfHJr19Bx/sUA7BjZQUtqNJdgW3YbrT0FEicsHj3PosW0b3yKnc0grS1lYaKx9nYkmaR9wI98Ct3D3VEfZhlTJTXUHCbSFCQwMakqmgIA9XU09unzGZ+CwTA1zKSQVMMkaHB6ieeCyKO2NqS1meZsZU2hOe0S7+wkdfQx5NespaNGh5WWOpuHnWGcoTSpTKFMU6hZsJg5Q9CWBmtu+RO+iBBfuJCm/gLpgq6U2j3cTfNw0ESnpTx5LSx2N5FQsGoCTWGC6qdSXxf5FGJGUzAYpgQjFA4gXM+nyR8gGZpQ5szBbpujzUdjSl0MFYZo7M9roXDM0Xj9/bQFQ0qdzduy2nQEOhw1JL5wAbaCQ7sVsXnzGEt84QLqe4fxlMewM8z24e0sGQic1UuWlI0No5GqFcODUU2hUtnsEKu+nlgQZGVX6ONsMBj2HiMUDiD6R4rUW2kaA2drrKWFeFv7OPNRwStQ9IvU9A0Tnz+fmmOOBaD2uW3YYtM93B2N3Zbdxrz+0XDUkEQQllpbgGRH57i5JBYsJLUjDUoxWBhkx8gOlg4miM3vxKotb6cZn9tO3cvPoO6006peW5jb0NlcvQ2hFZS6AIgZR7PBMCVMu0/BsGv+b91jfOy+z9OUuQyL0Sdiyxnm23aBpmGFn4gjtbUk53bQ9AS8UGI+yhQzxB1FYihHfH4nqSMOR+JxCmvX0rakrVxTGN4WJa6V9oUNcxUAkp3zx80xvmgR9kiBurxNupime7ibBX2QXLps3FhJJFj8jW9MeM2RpjBBqGmsoYli+L5CdzaDwbD3GKEww8g7Hv/v7psZqX+Cv0nezBHuaChoHVkyRYvGEfCb6xERYm3t2nxUUoNoqDgUmYri8+cjiQTJI4/UfoXl5bkK27PbWTAoxObOLXvCj3WOageJKuYjgLmDujpqd3Y7rTvzJM4aLxQmQ6gpTFTTKNbYOCoUjE/BYJgSjFCYYXzl7mdZpv7Ao0DSuYc39g+WbX+0plY7W1uaAe1stn0oDPRGY4YKQ7SFSWfBzb3mmKNJ/+xOOlJn8Fz6+Wjs1uxWzh6MlZmOQOcB2HPb8Xb2EOsYLxQSCxcC0DGo6Mv14fX0Ei+4JJct3aPrllgMiceRRHWhkGhsJuzzZjQFg2FqMD6FGcTT24Zw7/0KmaTOGHtoyYnwn11lf9m/+x5NIwqrVUf4hM5bv68/Ok6mmIk0hVhg+kkdcyz+8DCHplPsGBlNYNue3ca8Ho/EoeOf8BML9I0/HhTDKyUeCIW5aXh24Fnm92kPcKKC+WiySCo1oaYQb2wefW8czQbDlGA0hSnC933yrr/rgeF4Bd++9Yd8PHYrpycWkrQTrBt4liFRUU8EgIxfoHEY4q2twGhCmJRoFJlihva0AsuKbug1xxwNwCHbHHL1OTJOhsZEI9nuLlI5l+SyQ8fNKb5gAbnHHivLZg6xGxqwmpqYOzjEU/3rWaDbK5DYQ00BdDOeCYVCU1P0PjZBlJLBYNhzjFCYAlzP55yb3ozKpnnp9pMmtY8A/xn/ARsa5+GgeMOy1/GT537CI92P8IrFr4jGZQsZFo5Aol3f7O1AKFj9o9FHoaZgtbdFN9nE0qVYtbW0b0zD0TpXIWElqN2mNYxKmkLtySfhbN9etV5RYuFCOofW85OB9VzQp6CudsJchF3Runq1rqtUhbCnAkAiYcxHBsNUYITCFPDxu66nx3qGWIPi/w18hSa/YsHXcXhWgqfOupYlv/sab/rlGu4+O85DOx4qEwojQ30kPKhp00/v4U04MTgcjQkdzfH5o1FDYtukjjqK+g3dWiiM7CBmxZgfPOEnl40XCi1vfCMtb3xj1fnGFy6k4+H19Of7WdCntQSRyqWvJ8Ocf3rrhNvtkpBUk6dgMEwNRijsYx5/cTP39XydJssnbVv8+W8+y+sXnFFxrOt7jHh5GuO6GJxd187zz93G6esU1uPP8LrjD+fh7ofL9nF6tUM5GQgFq64OL2GTSudRSiEiZIoZDktD8ugFZfumli9n5NZbsXzFjuEdWFgs7FWomlTFBLVdkVi0kNbfeYiyWNgv1Kyo/pS/Lwh7KoDpp2AwTBXG0bwPyRddbr/zUnbE4cOHvZW2mjb+kF4Pc1867m9Tqpa3PHQtr/vze8m1LtHr69p4YfAFjt2mY/VP3JpiXf+6sp4Fbr8298TmzAF0yQmnuZ7GrEfeywM6PHVORhGfX550ljp6ORQKLOwVdozsYNvwNhb0QWzpIXv0hB9fuJCYp5jfB61DPokK2sa+pNR8ZNpxGgxTg9EUSnhh6/N84ZdXcWKugcVe/a53GEOhsJW75/ZzQmwBrz79vfz1vjR3bf41Ra9Iwh5NyvrFhl9w7f3X4vouRb/Ig9sf5MxFZwLw4s71LN5aAKBzXS/qeMWjOx6NTEiqX2ea2a2j9YW8lkaahofIFDPUxGpw+3qJeeXmI9CaAsCx/bXsGNmB67sc36eoPfbw3b5WgHgQnbRyQ5ARvRdO5skQtuT0RZvDDAbDvscIhYD1Tz3MZ/50OQ/VWtwXU/zDYI63Do4gwF9r4txXm+DCTJ7Di17VY3xmTh05qeXDZ36eFy++mPOTih+eO8xft/+VMxZqE9I3nvwGX3nsKxw/93g+cfonuPjnF/PHrj9y5qIzKXgFYs9txnYVNccfT+7RR5k7UsOD3Q+O+hUGdaxpqCkAyJxmmp/bwlBhiLm1c7G6tYmpNAENdE0iq7aWI3fGuX94B95wljkZSB26Z2afxKJAKLyghUIlv8S+xKoLWnYaeWAwTBkHpVAoFHJsT2+lvUZH7qx/4Jfcu/ZDPNRSy1X509h0RCs3brqLPxzyUtKFdNR/oOvwl/Ptc79d8Zi9uV7uuv3VvHHp3xL78JcYfvoZEsCxK2r5/Zbfc8bCM1jfv57rH7+e1yx5DZ86+VqGvn8z//FAI1+L34M65b94Mf0ih3fpMNa2K65gy9vexnn9C/nrjlG/gp3WOQx2EJIKYLXNofmx0UqpsR5tboqPKU8hlkXyqJeyeNvz3DGyg6ZubT3c0yf82Pz5KIGjtuieC6VlMqYCsW0KKRulJh/qazAYdo+D0qdw050f4623XcDHbjqe5796GLlHr+bmmlo+83+tvOyLf+KfvvIc1x/6QWpjtZzacQrfKlzCTd+uI/Xbv/L4zscrHvNHz/4I13N44x29DP/lL3T81zVYTU1c+kQD92y5B8d3+MhfPkJjspH3W+ex+Q0Xs/Nzn+el93YxZ/1O1g+s5/nB5zlyi4LFC6g77VSspiaO74qzvn995FeIp0co1MTKOo8l2ufSmIPMyCAAqT4dnhpfML5mUc3y5bRtzdKT6cbetB2A5KHjcxQmg5VI4LQ2EPfA6ZyDTEOROicVw7P2PMLJYDBMzEEpFFal2/jq13xO/HkN3+ydy5eddj5/k8XSp/ppufRSnK1baXvnp/nqhtO54qsv0viF71EzmOeyexQ3PvK/447neA4/XP9Drn5qMd5dv6Xtne+k9dJLaXnTm1jy+E7sbT188M8f5Km+p/j0uhX0ve1KVLHIgi9/Gamr5awnff645Y883/ccR2xVNKw6CbFt6k46kXnrelEofvzcjwFIDuUpNJTX/Um168ihrVuepjfXS13fCMWaeNTXuGzs8uXEih6N2zM0bE/j29ZePeG7nUGexJKp1RJC/LoaVNzYjwyGqeKgFApHn3cJ7e98JyvdTq78hc/Hvu/RFmvikO99j3n/dQ3L7vo5daefTt/1X8ft7WX+f/83i2/4Fk1ZRe2df2J9//qy4/1m029o2NDDqb/YROPrXkfbu98FQMvf/z1i27zuYfjVxl/xjq7Dab7ltzT97d+y7K6f03juOTSedx6nrxfuf+EP9D37JA05qD9hFQC1p5yCbN/JeYnj+c7a7zDsDFOTdXCb6srOX9+pbfu3/uXrvOKHr6BhoEChvZFKhM7mZd2K+X0KZ/6cvYrkaVpymL7Ww4/Z42PsDnM7ltJcO2fXAw0Gwx5xUPoU4gsW0H7Vu2l797vIPfY4uSefoOmCC6KOYbG2Nhb+z9fIPfYYqSOOiBycyZedyt8+8AA3Pfh1PvmaL0bHu2Xt93nPr21ic5qY95EPR+Gd8Y65NL3utbzyl3fx7KEpXvnjZ6k77VQ6r/1YdCNufsMbSN/+YxrvW4tr6SzdmuOPA6DulFMAuCx3HL+0H+Wmp29iadbFW1KuATTOW0w/8J5DLqP3uEM49Nb/ofmQyiahxJIlqFSSZd1FFvQqrOWLK46bLE1LDqeX31D/kiP26jiTxW5owEv0Tcu5DIaDkYNSUwgREWqPP445q1ePbyEpQu3xx0cCAaDzvf9GfU6Ruv03PDvwLABP9jzJsl88yfztRTo/+hHsxvIn9NbVq0kUfa66bZjEokUs+OIXy57Ma447DhbP58wnPRZvHKbYWBN1LkssW4bd3kbLU12ctfAsvrP2OzSNgGouP0dY/2jpDx/gtK/+mfptaeoXLal8zbaNfcRhHLZVMW8Aag/ds3DUkMTiRcFcpzYcNSR1+BEkli6ZlnMZDAcjB6WmsKfULF9O4uyzeN2f7uGtP3gDjZ2LmdfjcfW9PrXnvpqGV71q3D6pl76UutNPJ7d2LYuu/x/skqJuoIVP+0Vv4qgvfolFPQrv+MMiTUNEqDv5FIYfeIB3fvAb/PHnf6BxBPpbywVYbN48Gs57Dc6WLpzNW4gvWkj9mWdWvY76Y47lJU+sBaD1yL0z+zSccw6djkvNypV7dZzJMvff/nVazmMwHKwYobCbLHzv+yj+/k988yseTmIzHj7UpFjwXx+uvs91X8YvFsdpIyFNF1zAzi99mYa8IrGqvIBe3amnMHTXXTR/6RYuWXYSlrqfWEk4Kuin/4Vf/CKTpf6YYxniBwDUvGTvNAUrlaL5ojfs1TEMBsPMwQiF3SR56KEsvvE75B57HG9wEG8oTfOFF0YmnEpYdXVlZqixxOfNwz75ePwHHqHz1HJto/H1ryf3xJOkf/pTLnQcANrn712NodDZDJA0phiDwVCCEQp7QN1JJ1F30uRKYk+WRVe+h77aG6lZflTZeiuZpPPaj9H2risZ+N73yPzhDyw6+ZV7da7E0qWoVBIa6ycUVgaD4eBDwg5cByKrVq1SDz/88K4HGsax5V/egVVXy4IvfGF/T8VgMEw/VTNAjaZwkLLwK9fBXvQ+MBgMsxMjFA5SpqMkhcFgOPA4qPMUDAaDwVCOEQoGg8FgiDBCwWAwGAwRRigYDAaDIcIIBYPBYDBEHNB5CiLSA2zajV3agN4pms5M5mC87oPxmuHgvO6D8Zph7667Vyn1mkobDmihsLuIyMNKqVX7ex7TzcF43QfjNcPBed0H4zXD1F23MR8ZDAaDIcIIBYPBYDBEHGxC4Rv7ewL7iYPxug/Ga4aD87oPxmuGKbrug8qnYDAYDIaJOdg0BYPBYDBMwEEjFETkNSKyXkSeF5EP7O/5TAUiskhE/iAiT4vIUyLynmB9q4j8VkSeC14rt4A7wBERW0QeE5G7guWlIvLX4DO/TURmVRVAEWkWkdtFZJ2IPCMipx4Mn7WIvDf4fq8VkVtEJDXbPmsR+baI7BSRtSXrKn62orkuuPYnReT4vTn3QSEURMQGvgacBxwFXCIiR0281wGJC/ybUuoo4BTgyuA6PwDcrZQ6DLg7WJ6NvAd4pmT5M8AXlVIvAQaAf9ovs5o6vgz8Sil1JLACfe2z+rMWkQXAVcAqpdTRgA28mdn3Wd8IjM0jqPbZngccFvy9Hbh+b058UAgF4CTgeaXUBqVUEbgVuGA/z2mfo5TarpR6NHifQd8kFqCv9bvBsO8CF+6XCU4hIrIQeB3wrWBZgFcCtwdDZtV1i0gT8HLgBgClVFEpNchB8FmjS/7XiEgMqAW2M8s+a6XUn4D+MaurfbYXADcpzQNAs4h07um5DxahsADYUrLcFaybtYjIEuA44K9Ah1Jqe7CpG+jYX/OaQr4E/DvgB8tzgEGllBssz7bPfCnQA3wnMJl9S0TqmOWftVJqK/A5YDNaGKSBR5jdn3VItc92n97fDhahcFAhIvXAj4GrlVJDpduUDjebVSFnIvJ6YKdS6pH9PZdpJAYcD1yvlDoOGGaMqWiWftYt6CfjpcB8oI7xZpZZz1R+tgeLUNgKLCpZXhism3WISBwtEG5WSv0kWL0jVCeD1537a35TxOnA+SKyEW0afCXa3t4cmBhg9n3mXUCXUuqvwfLtaCEx2z/rs4EXlVI9SikH+An685/Nn3VItc92n97fDhah8BBwWBChkEA7pu7cz3Pa5wR29BuAZ5RSXyjZdCdwefD+cuBn0z23qUQp9Z9KqYVKqSXoz/b3SqlLgT8AFwfDZtV1K6W6gS0ickSw6lXA08zyzxptNjpFRGqD73t43bP2sy6h2md7J3BZEIV0CpAuMTPtNgdN8pqIvBZtd7aBbyulPrl/Z7TvEZGXAX8G1jBqW/8g2q/wQ2Axuqrsm5RSY51YswIROQt4n1Lq9SKyDK05tAKPAX+vlCrsx+ntU0RkJdqxngA2AP+IftCb1Z+1iHwM+Dt0tN1jwD+jbeiz5rMWkVuAs9CVUHcAHwHuoMJnGwjHr6LNaCPAPyqlHt7jcx8sQsFgMBgMu+ZgMR8ZDAaDYRIYoWAwGAyGCCMUDAaDwRBhhILBYDAYIoxQMBgMBkOEEQqGaUdElIh8vmT5fSLy0X107BtF5OJdj9zr87wxqEz6hwrbDhORu0TkBRF5JKhc+/KpnlM1ROTC0gKQInKtiJy9v+ZjmNkYoWDYHxSAN4hI2/6eSCklGbGT4Z+AtymlXjHmGCngF8A3lFKHKqVOAN4NLNt3Mx1PUAm4GheiqwMDoJT6sFLqd1M5H8OBixEKhv2Bi24l+N6xG8Y+6YtINng9S0T+KCI/E5ENIvJpEblURB4UkTUicmjJYc4WkYdF5NmgLlLYa+G/ReShoOb8v5Qc988icic6M3bsfC4Jjr9WRD4TrPsw8DLgBhH57zG7XArcr5SKMuaVUmuVUjcG+9YFtfIfDArZXRCsXy0iPxGRXwX18j9bModzROR+EXlURH4U1LZCRDaKyGdE5FHgjSLytuD6nhCRHwdZv6cB5wP/LSKPi8ihpf9jEXlVMI81wbySJcf+WHDONSJyZLD+zOA4jwf7NezqwzYcWBihYNhffA24VHQJ6MmyAngH8FLgH4DDlVInobN6310ybgm6XPrrgK8HT+//hE7/PxE4EXibiCwNxh8PvEcpdXjpyURkPrpO/yuBlcCJInKhUupa4GHgUqXU+8fMcTnw6ATX8CF0GY6TgFegb9Z1wbaV6EzdY4C/E900qQ24BjhbKXV8cN5/LTlen1LqeKXUrcBPlFInKqXC3gr/pJT6C7oMwvuVUiuVUi+UXF8KXbf/75RSx6CL7F1Rcuze4JzXA+8L1r0PuFIptRI4A8hNcK2GAxAjFAz7haB6603ohimT5aGgZ0QBeAH4TbB+DVoQhPxQKeUrpZ5Dl384EjgHXR/mcXTZjznopiQADyqlXqxwvhOBe4Liay5wM7qHwaQRkZ8GWkZYnPAc4APBPO4BUuiyBaAbqKSVUnm01nIIulnSUcB9wT6XB+tDbit5f3Sg9axBayzLdzG9I9DF5Z4Nlr875vrCOT/C6P/3PuALInIV0FxSrtowS9gdG6rBsK/5Evqp+jsl61yChxURsdB1fUJKa9n4Jcs+5d/lsbVbFCDAu5VSvy7dENRKGt6TyVfhKUpurEqpvxWRVegeAATzuEgptX7MPE6m/Po89DUJ8Ful1CVVzlc69xuBC5VST4jIanTtnL0hnE84F5RSnxaRXwCvRQuqc5VS6/byPIYZhNEUDPuNoFDbDylvnbgROCF4fz4Q34NDv1FErMDPsAxYD/wauEJ0aXFE5PASs001HgTOFJG2wJF7CfDHXezzA+B0ETm/ZF1tyftfA+8WEQnmcdwujvdAcLyXBOPrROTwKmMbgO3BNV5asj4TbBvLemBJeGy0SW7C6xORQ5VSa5RSn0FXHz5yF/M3HGAYoWDY33weXQky5JvoG/ETwKns2VP8ZvQN/ZfAOwJzzLfQJplHRTdD/192oSkH5Yc/gC7L/ATwiFJqwpLMSqkc8HrgHYFD/H60T+ATwZCPowXdkyLyVLA80fF6gNXALSLyJHA/1W/E/4U2jd0HlD693wq8P3AMRw754P/yj8CPApOTD3x9ovkAVwfmsCcBB/0/NswiTJVUg8FgMEQYTcFgMBgMEUYoGAwGgyHCCAWDwWAwRBihYDAYDIYIIxQMBoPBEGGEgsFgMBgijFAwGAwGQ4QRCgaDwWCI+P+AjopQ/JPgQAAAAABJRU5ErkJggg==", "text/plain": [ "
" ] @@ -248,22 +237,22 @@ "metadata": {}, "outputs": [], "source": [ - "evaluator = GymFitness(\"CartPole-v1\", num_env_steps=200, num_rollouts=16)\n", - "evaluator.set_apply_fn(param_reshaper.vmap_dict, network.apply, network.initialize_carry)" + "evaluator = GymnaxFitness(\"CartPole-v1\", num_env_steps=200, num_rollouts=16)\n", + "evaluator.set_apply_fn(network.apply, network.initialize_carry)" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "EvoParams(opt_params=OptParams(lrate_init=0.05, lrate_decay=0.999, lrate_limit=0.01, momentum=None, beta_1=0.99, beta_2=0.999, eps=1e-08, max_speed=None), sigma_init=0.05, sigma_decay=0.999, sigma_limit=0.01, sigma_lrate=0.2, sigma_max_change=0.2, init_min=-0.1, init_max=0.1, clip_min=-10, clip_max=10)" + "EvoParams(opt_params=OptParams(lrate_init=0.05, lrate_decay=1.0, lrate_limit=0.001, momentum=None, beta_1=0.99, beta_2=0.999, beta_3=None, eps=1e-08, max_speed=None), sigma_init=0.03, sigma_decay=1.0, sigma_limit=0.01, sigma_lrate=0.2, sigma_max_change=0.2, init_min=0.0, init_max=0.0, clip_min=-3.4028235e+38, clip_max=3.4028235e+38)" ] }, - "execution_count": 10, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -272,68 +261,33 @@ "from evosax import PGPE\n", "\n", "popsize = 100\n", - "strategy = PGPE(param_reshaper.total_params, popsize,\n", + "strategy = PGPE(popsize, param_reshaper.total_params,\n", " elite_ratio=0.1, opt_name=\"adam\")\n", "\n", "# Update basic parameters of PGPE strategy\n", - "es_params = strategy.default_params.replace(\n", - " sigma_init=0.05, # Initial scale of isotropic Gaussian noise\n", - " sigma_decay=0.999, # Multiplicative decay factor\n", - " sigma_limit=0.01, # Smallest possible scale\n", - " sigma_lrate=0.2, # Learning rate for scale\n", - " sigma_max_change=0.2, # clips adaptive sigma to 20%\n", - " init_min=-0.1, # Range of parameter mean initialization - Min\n", - " init_max=0.1, # Range of parameter mean initialization - Max\n", - " clip_min=-10, # Range of parameter proposals - Min\n", - " clip_max=10 # Range of parameter proposals - Max\n", - ")\n", - "\n", - "# Update optimizer-specific parameters of Adam\n", - "es_params = es_params.replace(opt_params=es_params.opt_params.replace(\n", - " lrate_init=0.05, # Initial learning rate\n", - " lrate_decay=0.999, # Multiplicative decay factor\n", - " lrate_limit=0.01, # Smallest possible lrate\n", - " beta_1=0.99, # Adam - beta_1\n", - " beta_2=0.999, # Adam - beta_2\n", - " eps=1e-8, # eps constant,\n", - " )\n", - ")\n", - "\n", + "es_params = strategy.default_params\n", "es_params " ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 13, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/rob/anaconda3/envs/mle-toolbox/lib/python3.9/site-packages/jax/_src/tree_util.py:188: FutureWarning: jax.tree_util.tree_multimap() is deprecated. Please use jax.tree_util.tree_map() instead as a drop-in replacement.\n", - " warnings.warn('jax.tree_util.tree_multimap() is deprecated. Please use jax.tree_util.tree_map() '\n" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "Generation: 0 Performance: 22.3125\n", - "Generation: 20 Performance: 40.8125\n", - "Generation: 40 Performance: 83.0625\n", - "Generation: 60 Performance: 188.6875\n", - "Generation: 80 Performance: 198.5625\n", - "Generation: 100 Performance: 200.0\n", - "Generation: 120 Performance: 200.0\n", - "Generation: 140 Performance: 200.0\n", - "Generation: 160 Performance: 200.0\n", - "Generation: 180 Performance: 200.0\n" + "Generation: 0 Performance: 21.875\n", + "Generation: 20 Performance: 44.625\n", + "Generation: 40 Performance: 194.4375\n", + "Generation: 60 Performance: 199.3125\n", + "Generation: 80 Performance: 200.0\n" ] } ], "source": [ - "num_generations = 200\n", + "num_generations = 100\n", "print_every_k_gens = 20\n", "\n", "es_logging = ESLog(param_reshaper.total_params,\n", @@ -362,7 +316,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -372,13 +326,13 @@ " )" ] }, - "execution_count": 12, + "execution_count": 14, "metadata": {}, "output_type": "execute_result" }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAADgCAYAAADsbXoVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABiHklEQVR4nO2dZ5hV1dWA33X79N4Zekd6ExRBxEYUe4+KQY2KPZrYEv1MYtSoiS3GjgUFFcUSO4qi9N7LAAMM03u/dX8/zpnLDMzAAFMo+32e+8w5u65z7p2zzt577bVEKYVGo9FoNACW9hZAo9FoNEcOWiloNBqNJohWChqNRqMJopWCRqPRaIJopaDRaDSaIFopaDQajSaIVgoaDSAi40Qkq73l0GjaG60UNG2KiFwpIktFpFJEckTkKxE5+TDaUyLSvd75OBEJmO1XiMgmEbmuZaRvUoZpIvK3JvLOE5GVIlIuIoUi8oOIdBGR/5oyVoqIR0S89c6/EpHO5rWt2Ku9eLN85n7kUSJSZba1W0SeERFrvfzLRWSRWSbfPL5FRKTe9XjM+sUi8p2I9DbzHtlL1koRKW2J+6g5MtBKQdNmiMjdwL+Bx4AkoCPwH+C8Q2jLtp/sbKVUOBAJ/Al4VUT6HrTAh4mprN4G/gBEAV2AFwG/UuompVS4KedjwMy6c6XU2fWaCRWRE+qdXwlsb0b3A822TzPr3GDK9AfgWeCfQDLG93ATcBLgqFf/SbN+ByAfmFYvr76s4Uqp6GbIozlK0EpB0yaISBTwKDBVKfWxUqpKKeVVSn2ulLrXLDNCRBaISKk5inhBRBz12lAiMlVEtgBbRORnM2uV+cZ6Wf0+lcFsoAToKyJOEfm3iGSbn3+LiLMJeVNFZJaIFIjIdhG5/RAuexCwXSk1x5SlQik1Sym18yDaeAe4tt75NRiKplkopTYC84AT6n0HtyilPjLlUUqpFUqpq5RS7kbqVwPvASfsnac5NtFKQdNWjAJcwCf7KeMH7gLizfKnAbfsVeZ8YCTQVyl1ipk20HxjnVm/oIhYROQCIBpYAzwInIjxsB4IjAAe2lsIEbEAnwOrgDRTjjtF5MzmXWqQ5UBvEfmXiJwqIuEHWR/gXeByEbGao51wYFFzK5t1xgArMO6pE/j0IOqHA1eZ9TXHAVopaNqKOKBQKeVrqoBSaplSaqFSyqeUygReBsbuVewfSqlipVTNfvpKNee5C4GHgauVUpswHm6PKqXylVIFwP8BVzdSfziQoJR6VCnlUUptA14FLm/epQavZxswDkOxfAAUmvP1B6McsoBNwASMUcI7zay3XERKMJTba8CbGMq2wXcgIvPNkVmNiJxSr/495j3MwFBEk+vlXWrWqfv8eBDXoznC2d+8rEbTkhQB8SJia0oxiEhP4BlgGBCK8ftctlexXc3oK1sp1aGR9FRgR73zHWba3nRij2Kpw4oxDXNQKKUWApcCiMhwYCbGiOX+g2jmbYyH8miMt/6ezagzRCmVUT9BRPb5DpRSo828LBq+JD6llNpnFGXygVLqtwchv+YoQo8UNG3FAsCNMf3TFC8BG4EeSqlI4AFA9ipzOG59szEe+HV0NNP2ZhfGWkB0vU+EUmriYfSNUmoJ8DEHPz8/C/gNsO0g1yP2pu47OOiFfc3xg1YKmjZBKVUG/AV4UUTOF5FQEbGLyNki8qRZLAIoBypNE8ibm9F0HtC1mWK8DzwkIgkiEm/K824j5RYDFSLyJxEJMefzTzDf9JvCKiKueh+HiJwsIjeISCKAeU2TgIXNlBcApVQVMB64/mDqNdJOKcaU2X9E5GIRiTDXXQYBYYfTtubYQSsFTZuhlHoauBtjcbcA4438VmC2WeQeDPPJCow5/Jn7trIPjwBvmXPblx6g7N+ApcBqjIXn5Wba3nL6gXMwrYcw1iZewzArbYr7gJp6nx+AUgwlsEZEKoGvMRban2yijSZRSi1VSm092HqNtPMkxnfwRwyFmoexdvMnYH4zm7lsr30KlXWKT3P0IzrIjkaj0Wjq0CMFjUaj0QTRSkGj0Wg0QbRS0Gg0Gk0QrRQ0Go1GE0QrBY1Go9EEOap3NJ911lnq66+/bm8xNBqN5mhj702hQY7qkUJhYWF7i6DRaDTHFEe1UtBoNBpNy6KVgkaj0WiCtJpSEJF0EflRRNaLyDoRucNMjzXD+20x/8aY6SIiz4lIhoisFpEhrSWbRqPRaBqnNUcKPuAPSqm+GIFNppoBP+4D5iilegBzzHOAs4Ee5udGDI+ZGo1Go2lDWs36SCmVA+SYxxUisgEj2Mh5GIFHAN4C5mI44zoPeFsZzpgWiki0iKSY7Wg0h0UgoHjg+7dYmb8Eq7joajsXlyUGgAL/Kkr9m4m19qFaFWDFSYr1RIoC68jxLeTwvHU3xB6oxUIAtyUUq/LSJ2cTdp+iODKEWpeVKns0ldZoQHAEaoj15lJiT8RtaTknpsnu7dRawii17/Fh5wpU4seGEgsW5Q/KqJo2UgEg1tqHNNsYSvybyPL9tJ+SQpJtGHGWvmT6vqY6kHfI8odakuhkOxObuFBKsdX3CTWBhkYnFrERIvGkWEcTYokPpvtUNbt8P1AZyMYqLnraL8UmLvzKw27fPGziItV2UpN9+1Qtuf7FVAaySLQOIdbaG4DaQDHZ/vl0tp2FpV74cJ+qYZv3c5KsQ4mwdKJKZWPBTrXKo8i/ns62s4O/Q5+qplaVEG5Ja9Z9GJgezW9P7HTgggdJm5ikikhnYDBGGMGkeg/6XIzA4WAojPoBVLLMtAZKQURuxBhJ0LFjx9YTWnPE4A8oPl+/hpXZmXQIObRQwT9t3c4K9RyCDcTLrtrFhBTfiMUfRWXSi2CpZqtvT5TKtd7ZBOzZEAhFAo79tHxgbD7F+DW1jNxaTWS1Ii8OvhhtIbFUMeWjPQrHbYeSCNiaJrx5eiguaw01BIgq3k65y0WRvWHANktAoQREIHCAh3dQFvyk5pZw/kKIqBQevjyGUdsquXKOh13J4PBAQgm4HfD1SCvf9ItGIQgKF25qcQVVpLLUkO1Zxva83tTEzsLv3IT4I/bp04LCb/Gy2/8TBELBUo34o4O61uUJcMGCGgbs8LCms533TjGu04EXB16qCNmjnASUtZSM2u8IKboeZa2kJu5jo19l3dOpeFHWKraVbSek9CoC1hI8YfPwhi40+49AWSsoyOuI1dOFqoSnUbZiUFa27kzDEthzr70hi/GGLCe0+Cbc4V/jifwalJDp/RpX6WXYa0ZSHfsKftd6dhQEcFSPDtb1hM7HHf0Z23yfgbKBNIwvtbu0Glf5+SgUNXEv4bfvIDz378bv9AC47NYDljkUWl0pmKEHZwF3KqXKRfb8eJVSSkQO6jVMKfUK8ArAsGHDtIvXo5S7vnyJ3YUuYqQ/SsHu2vXsZjaukquwBKIblK3y+HHHvY4tbBNV2+5GeeMOur+QxO+xxXmZPelDPAEPt865leqQ/zAgYQALc2p588y3qPJW0SGiA0tyl/D00qc5q8uF3D/iflw2V+ONBgLGE7nuN12yA9Z9DCdOBZuhSNzbtrH7rrtxb9qEI8KLPTWNzttLOTHTBwpsnTsQf8fdeLMy8eXkELdpNcnLNnE+OcQMDKWgfAIlX/xERGoJaY9eT86nu8BuI/rCi8i+507EXULq0GxCpr4Dvc4OiubLzSH7jikkXDIOR7ifXU++R8Jtt+IKK2fLM68jNjsBt4/vV9dQvKwKH6EkVoZjcdpxDkygZutupnxXzD8GdyLnFxfVixZiC6kl7dYLcV71TwA+2PQBf134Vz68tTdXfZnLmLRz+NvJe3kin/NX+PVZvBf8h7coZ17WPG4dfCvDEwZDZS5EdSD3oT9SsvhzbKE+ui728YeH38PyzZ2onYuxOQNw7rMwdHKwyRX5K7jnp3uIiP2EKGcUWZWJfHXhVzisDZX3Ld/fQm51Lh/ffBoXfHoB28u2MyH9VKb0n0J6RDonzziZqWdG0zEigtt/LGbqoKm8uPJFrp9YwPX998Qh+sPcL/l2x0a+unsYf134FesKOzDjnBnc+9O9LJD3GTOokO92rCfEFkJ02jz+d8F92K12AG7/4VM2FCdzZe8rKawppF9cPwIEiHJE8cGmD1hduJrvb36GL7d9yUO/bgbgP1MSGZw4eL+/53WF64hxxey3zKHSqkpBROwYCmG6UupjMzmvblpIRFKAfDN9N5Ber3oHM01zjOHx+fku9w0svmRSq7vhse6kKOJ5AlJLp/TtdLSfTlUgh1BJRkSwWALMqc7EHfBzzrjlPHbSPw+qv1pfDRd88TiDE8fRLaYbANN/M51b5tzC/Oz5XN7rcoYk7bFr6BLVhYt6XITVsp83saKt8M4FcMJFMOFhyN8I75wPFTmQ0Ad6nQVA9l1T8e3cSYdTSoiYcAZc8hbubdvYOfk6/OXlpD73Iq6eDaNr7r77bgq+/ZbinWH4K37G1acvFevWkfePxynbYrzBln00C6vLj1gsZH6fQLr1HsKfOAUcYaAUBfdeSdWqXHw7NxGa4KEmO5zSt14hvGcEyi90eu9dcu65nfx5Ofg9dlL+7wGiL9sTgrpm1SoyL7ucrNcWUJ3nJDytlpriEHY/P5vOp/4OS2of+sX1A2DOzjkU1xbTL77fXje+DBa9DCLYZ93I9ZdM4/qz34Jdi/E+czJlS3fjuu7flHz8BdHdq4i78Sa2/ult8m+7gOrsANa4fnQ5PQfZuaiBUhicOJhHRj3CLXNuAeCeYffsoxAAesX2YkH2AvKr88kozeDOIXcypf+UYH60M5rM8kz8yg/AlX2uZGneUmZumklWRRY2i42HTnyIHeVGBNfM8kx2lu+kc1RnopxRvDjhRf628G98vOVjOkV24g9D/8DtP97OVV9ehdvv5plxz7AwZyGTuk3iuhOu20c+X8DH3Ky5TFs7jbfWv0Wf2D5sKN7AsrxlDE4czIaiDczaMovfdP0NgxMHszxvOclhyaSGp3LfvPvoEdODZ8Y90/Rv9BBpTesjAV4HNiil6kv+GXCteXwt8Gm99GtMK6QTgTK9nnBsMm97BmKtxeLKYvatI4jp9AmJ4dHEueLokJrDlWMDzKu9l7NGFvLPSwZy9Vg77kA1feP6MjdrDrO3zcBpV0S47M36LCuYT5m7lGv6XhOUITksmbfPepv7RtzHHUPu2EfGoELwVEFlAXxyMzw/FLXtV4qe+gs7Lz0HX95OWPgSlO6Cdy9EBQL4vCGw3Zhb9+3KoHZTJjH9IOKSG2HSCyCCs1s3Os/6iM4ffLCPQgBI+vOfsSUm4uzZiy4fz6LT229hjY2lZEs4rngfnU8vJnpQOF2eupOu3/6Es0s6u7/34f7oYQBqZz5M6dIcnOmxuEsclGwOR2wWKjMqqVi+A2uEE1f//sTedDt+jxVrdBSR553fQIaQgQMJGTqU6jwnrk7xdHjuOVL/8RjuUhu7r7+S6gU/0cPjwS42Ptj0AQAnxO01tbf0TfBUoK7+jOxV6VS+94xxP9+9iNzvSihYFc6uOx/CYhcSTknAMekBIod0pHyr4Ku14s4qojrQF3btG6huTIcxXNjjQhJDE7m458WN/s56xfTCp3x8tvUzAAYkDGiQ3zGyIzvLd7KtdBvxIfFEOiK5ovcV5FblMmvLLD7a/BE1vhp2VhgRUDPLMsksz6RTpDGPb7fYeWTUIzx28mM8M+4ZxqWP49T0U/EGvORU5XDz9zdT46thTNqYRuU7ucPJxLpieW7Fc7isLp4a+xTdo7uzNHcpMzfO5NIvLmXmppl8sOkDAirA1DlTeWHFC3gDXrIqsugc2bnRdg+X1rQ+Ogm4GhgvIivNz0TgceB0EdkCTDDPAb4EtgEZGFG3bmlF2TTtyA/bVgHgVz6+zvyazSWbuabvNYxKHcXSvKX8b9v/APhi2xcALMwxHgrPjHuGEckjeHLJk1zz1TXUDxC1sXgjZe6yRvtbW7gWh8XBoMRBDdLDHeFc1ecqwh3hjdbju4fhsVR4qjus+QB/VTW7rrua/Nc+pGo35GafhvLWwJsTKVlRQsZnCWz5OIbahXMAqHzzL0Y/v38STn8UXJHBpu2Jibh67asQAGwxMXT//ns6vfM2rt69sYSFkXDXnUhoKMnPTyfkn1tImbEE+4RbsMYnkf7Km4jDwe7nP0NVFJD38kwsThsdZ35ByMCBSEgIyQ/8iYDPQkWWi/CRQxCLhchzzsHesSOxU6Zgce07RRZ/001G3af+g/Q7l/Czzifxt2dStbOGHdfdROkt59LT7WZnxU5sYqNnbL3r8ftg0X+hy1jK1xRStsFH0c9ZsPAlKra5qdxpIbZ/gNiTUkkZ7cbW80Sjz788i7NrKun/eRFLZCSlG4HibVCZv498j4x6hC8u+IIwe71F+Nw1UFMCQM8YQ55Zm2dhEUtwZFNH58jO7Cjfwfby7XSNMiK6npp+Ko+OfpS7h96NX/mZnz2fGl8NAEvzllLjqwkqBQAR4dxu59IzpiciwnPjn+OT8z5h6qCp5FTl4LQ6GZEyotHv2W6xc12/6xiUMIh3J75Lx8iODE0ayor8Fbyw8gWGJw9nRPIINhZvZGf5Tiq9lWSUZrC7Yjc+5aNzVOdG2z1cWk0pKKV+UUqJUmqAUmqQ+flSKVWklDpNKdVDKTVBKVVslldKqalKqW5Kqf5KqaWtJZumfVmVtyF4/OLKFwEY12Ecw5KGUVxbzKcZnyII87LmUempZHHOYrpHdyctPI3XzniNqYOmsq5oHdlV2QD4A36u+eoa3lj7RqP9rStaR+/Y3tgt9uYLuWsx/Pos7tjx7NpwIrnV17Br9SCqClwkXzeBxLtup2LResqqhuAv3EXeilhsSamIzUrJ0lzYMZ+qXxZgjXDgOunsA/e3F2Jp+K8Zc8kl9Jz/KyGDRzRQLgD2tDSS7/o97hIru6+eSHWOhYRrLsAWG0P6q6/Q5eNZRF58ORanMVscPvEiACxOJ92//Yb4G25oVIbwMSfTa+kSQvr3D6bFPfRven74ItEDQijaGMElP3kA6BHTA4eqt9i9axEVG0qoChlLwXPPA1Bd4MD75RPkrYrH0a0riReOJKnreiKTCiDdeHA6e/Wh65dzCD91PFHnn0f5su34ai2wa9G+90iEEFvIngR3Bbw2AX78B2CMBJxWJ1mVWXSL7kaoPbRB/Y4RHcmrzmNLyRa6RHUx7olYuKDHBUzoOAGAbzK/CZb/OetnADpFHNji58o+V9I7tjendDiloYx7MfmEybwz8R2Sw5IBGJY0jGpfNaXuUv4w9A8MTBjI9rLtrCxYCUBm2Xa2l2035Ihsecsj0DuaNW2MUopdlVtxEUf36O7kVuXSLaob6ZHpDE8eDkCtv5ar+16NJ+Dhw80fsiJ/BSNTRgLGg2Bsh7EArMo3RhwFNQXU+GrYVrYt2M+agjVc+9W1lLnL2FC8gT5xfZoWavGr8NntxsJxIADbf0bNnkrJ7hS2v7Gd6swySr/6kZqVq0l76ili/vQ8sTf8HteAARQuD1BROwDlC5D0wANEjD2R8h0h+KddQmWuk/Bx4/d5wB8qjb3N1xFx5S2EpNqo2FiJIxpipj4EgDUyEmeXLlgcDsLHTwCbjbAxpzS7T7Huu65i6XMayTOXET52DD3XGfmXfp7J9nGDUbUVAHgWfETWL7Hs/MtreHftIuHuuwFh97xIvOWKpD/+Eek2BrzVRqPpI/fpJ/rii8Hnp2J3eKNKYW+8i2az64dQyr/9AQDbktfpHmo8bPvH99+nfKco46Fa46sJKoU60iLSCLOH8dMuYyqwT2wfimuLG9TbH3aLnXcnvssTY55ovEB5NrxyKuxe3iB5aNJQAManj6dffD96x/bGr/zBUXONvzY4cj4ap480mgZkl5XwS0Y+Xms2aWFdGZJoLO6OTTce8ukR6SSGJBJiC2HqoKkkhyXzzLJn8Aa8nNn5zGA7PWJ6EGILYVWBoRR2Vxr2CLvK91g0f5LxCcvzl/PK6leo8lbtM3XQgJXTYflb8M39MG0i6s1z2f2/EnLnQeiQoXT9/DN6/PwTXb/4gsiJEwHjTT72mmvwZudTsBhsKSmEDBpI9FVTCHgt7PzGRcAjhJ92Vovew6YQi4WkW36LLcRP0uSzEKdznzKJ991Hx9dfxxqxr9noQfcnQuiIkdhqLITWKrpsKsFdGKD2nT8BUPmDMYWWeO89JN57L3E3XI8jKZqaIgchA/oRdsop0Nmca3dFQfy+U2nOHj2wp6dTWRwPaz+GjO+blKdm3Tq23/VPKrNd5PxQgS9jCXz1J3pVlgJwQvy+psz13/i7hDXcG2ARCz1jelLtqybEFsKIZGMk47A4SDYVzYFwWp1BK6R9+OZByF4OGXMaJCeEJvDsqc/y51F/BqB3rLEPYlHOIuzmdOmPu34k2hlNlDOqWXIcLFopaFqdouoKzp9xL2d8PJ4bvrkdi7OAgUm9GZ5ijAxO63gaYPjyndJ/ClMHTSXUHspdQ+7i8l6X8/kFnzcw0bNZbJwQf0JQKWRXGtNIuyp2EVABlFL8svsXAN7b+B4AfeP6Ni5cwI9vxyY8NRHGHHjOairTplKRaSf+lltIf+1V7ImJWKOicHZt+DYZecbpWBPi8RUUEHnWWYjFQuiJJ+JIjsZd7iL2d78jYsJpLXYfD0TI+XfS/cXbCb/u0Ubz7UlJhI1sfH77UHB0Me7H6xuLCC81pqbKv/gCFr1C5ZYKHCkxxE2ZQtyU3yEiREwyFoQT7/0TIgIJvSAswRglNDKaEhHCx46lKitAIGCBdy+CJa/tU075fOQ88CCCl7QJEPAKefffRlmmixN25QIwIH7APvXqT790fecS2NZw812vmF6AMc1UN5JIj0jfv1Vac9j+s2G6DFCwYZ/s8R3HEx9ibLjrENGBUJsx7XVKtbG2kVOVQydLKGz+9vDkaAKtFDStzkNzXmWr+2tibd2wR6xHJMCIzDmcsfFnPjjnAwb8+BT8Xww8lsqVCSO4tp9hnDax60QePPFB0iPS92lzYMJANhVvosZXE1QKnoCH/Op8tpVtI6cqhy5RXfAFfDitTrpFG6aoVBXBJzcZb2q7FkNJJrmLnGz9PJIiz3mo63+geH42tpQU4m+5eb9TP+JwEHPpZQB7RhAidJz5Kd3m/EjSH+9FbG0YssTmREbfCs7DHwk0hzqlkOzuTcANWK2UZ4Xh//SPVOc7CT91fIPycb//PemvvUbocONlABG4ciac3cQUCxA+dizK7aF6yL+h23j49i+w6WvUx7dAwSYASj/8EPemTSQNKibymruJ6V5L+ZoyshfGcOJ8L/8adDe9Ynvt03aoPZSEkARCLA6SvB5Y/UGD/Lq39E6RnYKLuoc9j++uhM/vgOhO0HUc5NdTCjvmw09PNihuEUtQ9lE1tcT6DfPZTiVZexRLC6OVgqbV2V62FfyRzP3tB4xONXZ79sxaiWXJa/QpL4D1n0KXscb88o5fGlZ2Vxj7AfIbvlENShiET/lYV7hnwRmM0ULdKOHR0Y8iCL1ie2GzmA/nldNh1fuw+BX4+EbIW0ttiR1xOMj/eAk77v4b1YsWEXPlFc16oMfdeAMd33yDkP57pifsSYnYkxL3U+vYwNGhA9jtVBQa1xp1wfn4qiB722hUQAg/c1KD8tbwcMJP3suFRNpQiO3aZB+hI4YjoaFU/PwLTHoeLFYq/3UNmx+eQ/mb/yDg8VDwr2cI7egkokMt9D6HxHN60vHUQpxJIahKCxM8gSbb7x7dnZ6OGGO/9KYvDaspk7qHcafITsH5+04eDzw7CL6+H6qL9zQUCIDfe8B7xjf3Q/F2uOC/kDIICrfsqffTk/Dj3/extKobsfT2eOjiMcp2ri6HxCZGv4eJVgqaVqfYs5tQScYiFp4Y8wR/PeEmOhZ58VV7YObVYAuBi98AZxTkrGpYeddi2PoDbG4YYa/O5nxlwUp2V+4ODrd3lu9k3u55dI/uzqDEQUzpP4XLe+3ZlMW6T/DHDoAzH4OS7QRWfYK3ykrc7yaT/PBfqF2zBnE6jUXOZmBxOgkbNerQb85RjNjtODp2pHb1agDib7iBkIEDqVy5HWtCPKFD9r8rtzlYnE4ixo2ldMZMdt71MDl5Z5D1awIBr4WK+aupXToff3klMd0rkfNfhMgULN1OIizJg6PnCXjdzj2L1DUlMO9p8NYG23/0pEd53NeVLZ8lUrahBnbOD+b1jOnJ2A5jOTX9VOJC4nhw5INcUu1GlWaR9cz7lL9wt1HwvcvhbwnwRJdGTWeDbP8Zlr8NJ92BP7Y/WW+vxl2ijI2QNaWQOc8olzkPVrwLL44En4dT00+lb3g6KettDN9pjhTcXkjazzrZYXBUh+PUHB3UkEdHpzFlEO2K5nx7Alt/jsWV4CBt+G4YOpmAclLr60FI9sqGXnzy1hp/8zc2aDPGFUO3qG4szVtKTmUOgxMH8+OuH1lVsIplucu4pp+xUa3BxrSSTCoWryfr11hicjeTaAXPov8B8Th79ibyrLMIHTECf1kZtpjWcSFwrOHs2gXP1q1YoqKwd+xI55kz8FdWgQog9oMwAd4PyY8+irNXb0o/+oja8nJCBg5BqnZRnZmN46sZgCLslpfhBHP95sSbIa4bti93ULV4Gew0N78te4vAt4+C34aMvR0RITksmcrtu6motpG9KBqZ8QqR9xvWWQ6rgxdOeyEox+W9L4df38AdMpiKnVmoeeuIvNttvLAk94fc1caDv38jLxRKwZxHITINxt1P6TvvU7FkI67+LpwFGyBvLUXrnZRui6XrkJ+Q7BVQsBF2L2N0p9GM7jWFTSsfYGCpD3pB+uwIdud8QtqzLb9mpUcKmlZlV2kRWCtJr2fpobJX46mwUV0SgbKHU5LXjS3jTmXH9GyqV25uOAzPNZVCwUb2ZljyMJbnLSenKof0iHQ6hHfg862f41M+zu167j7lAytnkbciEmtEBCUff0H2io64S41FQ2e3bsG/oUN0KI/m4uhiTP04u3Wjzq+ZNTysRSyc6rCGhxP/+xvp/t239Fy0kE7vvkPEKSfhq7FS9s3POGMV1r7j9lQIT4TBv8WenELAHcBfsJPA7vXkvvgOmz5KYdNN/2XbmWdS+soToBS+PGMx2hbpJH/WEqhowoOrUlCwicp8Y7Nczc5KVGEGucsiKMwfiHJEQuYvjdfd+D/IWgJj/4iy2CmZPh2A2jK7MTW68X9U5kXgKbfhXfiZoWAAts01uq4oJOC1kFoV4M+FxVjdViyRsYd7axtFKwVNq7J4t7EY2KvevLEvYzUowVdcjvfaxeS/Mt1YtLRYqMoVKNy8p4G6kULhZmPeth7Dk4dT46vBG/CSFp5GekQ6PuWjb1xfusd0byiIUpS8Ox1vlY3Uf/2LmKuvpmK7n5oiO1gER6fW2Qh0rFO32Ozs3v0AJVuW0NPOB8Bbrgjt3QEasQiyJRsOmH01TrJumEzJimqiB8WS0L8cS+V2cp6ZRs3/XsdfZOyAjjr3XLyVFvwf3wFluxuuGQBUF0FtKVXbjL0VfjdUfvUJJVvCKXhvDrnrOqK2z9tTviyLwA9PUXj9CALTfwux3WDQVVT88APe3buxRkXhLg+FjO9Rm74zfotATU4NiMVYazGVgi/fUFziC+GS8kr8tYK1lUazWiloWpU1eRkADE7pEUzzZG4JHpe88w6B8nLifncdrl7dqC5w7FlX8NYayiAs0ViELtvZoO1hScOCxylhKXSMNFypT+o2yfin/u7h4KjDt+ILChdVET64O+EnnUTkmWeAX1G2PRRHaiLiODz32McrdWa6zu7d2rRfR/9hWM2NwmEnj2u0jD3Z2E/gSTyVqsxyYntVkvL828RfdhZpFxovAe4l3+Cr9CJOOyGjx5tp38O/+sLLpzRYeKZwMwGfUL05h/Ahxu85//VPAIg8+0xKlxXj2Z5puOX46j54bjClrz9NwS8VlKjz4LovURYbxW+8iT01legrr8BTpgjsWI67KhTlMdYLaosdVMkwdnwXSdaMzdSsWIy/0Bi9+L12Al4BhVYKmqOTjJLtKCUMTzOVQk0J3vySYH7JzJlgsRA2ahShI0dTW+QgsGuFkVmwEQI+OOFC83xTg7bjQoxd0QBp4Wn0j+9PhCOCs7ucDRs+h1//HRzOFzz9OAG/kPh/hofVkMGDsUZHE/BZcPbed7erpnm4+vUj7uabgia5bYWIENotAVCETrym0TK2JEMpVFV1goDg6pwA8d3h4tex3WrsEPZtXo6v1oItNhKn6ZzQ3eFyGHkTlO0ynBsqBZ5qKNxMVZ4D5fMRc8l5WJ1+PIXVhCQq4n5/EwC1pTZ44yxY/DIMuIxyn2FtVfzLDpQzluoFC6hZsYK4G2/A1bcvKHD3vo2aEx4yZU6ipiaFwrWh1GZXULHbSdk7r+IvMoII+WsC+N3GY9sWe5QpBRF5Q0TyRWRtvbSZ9ZzjZYrISjO9s4jU1Mv7b2vJpWlbsqt2YfXHEVa3wzZvPd4qG4jg6tcPVVtLyMCBWKOiCB0+HBUQapcuoGr+fAJZK/F7hNyfavFWWfesKwQC4DN87gxLGoYgpISnMLHLRH669CdiXbFQbnpdz/ge97IfKV1eRMy4vjh7GrbnYrUSPm4cAI42fss9lhCbjcQ77sAWH3/gwi1M3D2PkHTzpViTG5/6sycmAFC12Bh5us69PZhncbmwhjnwlnvx1VqxxcdjT03FEhqK25sME/7PsIZb/QF8OhWeGwxZS6kpDgWbjdCxZxESb/wGI06Ix9G1K1ituCvDoKoALnwV7/AHqFm1jtDhw/Hl5VE8/T0KXngRW1ISURdeiKu3GbXN0Z+adZuwxsQQceYZ1OTUUr16M3FTpuCM8OPN2oavpBSAQK0PX40xVdZaI4XWtD6aBrwAvF2XoJS6rO5YRJ4G6ru13KqUGtSK8mjagVJfNuHWFJRSxkJk/nq8VVZsCfGEDhtK7bp1hJm26yHmAm/Wp/n4Z04hvG8SqjSequxvsQ1NIL5upDD3MVg1A25bxo0DbmR06uig07GgW4EK0+t6xhxqft0CSoi9488NZAsffypls2fj7Na28+GaliHkxPGEnDi+yXxxOLDGx+PZscMwnx17VYN8W1IC3upyfLUWnEmpiMWCs0cP3Js3o5SFQJeJWNd8AGa8BVZOx1PbEUfHDlhiUglLs1CZo4gc1Q+Lw4Gjc2fcLjs1Q85i143P4Og0C4CUv/2VrFtvJf8JY5Ne0kMPYXE4sKelYQkLw71xIzWrVhnuyvsPoOTtd8BiIerCi6ie/V98hSX44/dMb3o6XQr8cPRNHymlfgaKG8szYy1cCrzfWv1r2p9AIIBH8jktQ7HlxJEEamogby2eGif29I7Bna11b+y2mBicXdLx11qJGNaNyvV5VGXbkZAQqovDIX+9MUJY+oYxtF//GQmhCZza8dR9Oy83lULBBrwbFoGAvVtD/zcR48eT9Jc/E3H6hNa8DZp2pG5dwdG9+z4msvb0LviqrfhqbdiSUgFw9jSUQtZdd5P56kZDIaQNhV4TQQXwVFhxdO4MIsSc2IFuE/OxdzOmH509euDOq6Z8bTH+8vLgg97RqRPpL79M2nPPkv7Ky8RcYeybEYsFZ69elH3+BZ6tWwkZNJCQAUZbYSedhD05GVtcNL6yGvzlVUG53dIZOPbWFMYAeUqpLfXSuojIChH5SUQaj0qhOarYXpKPWNwMyc7FX1aBZ/1SY/qo2oGjQxrhp51G16++JKTfnk04yX/7Bx3OcdGhz3JSTywh6YYLiTr/PGpyvKisFfDdXwwrEFtIo35wgpTvDu749FaCLS52n4eC2GzEXnnlfr2Pao5u6iyQGgtmZE/riKfKTsAj2BKMqSZnj574y8qonDMHz+48fCf9mRLLRWz/sAoVEDzFHkMpAJLYHUe4H+J7mHW74921i4rv5xA2ahRdv/wfac89Z/SVmkrkGWcQfsopDTzPhp86DktoKDFXX03Mb6/G3rEjMb/9LQm3TjXkT0rCV63wldcE63i2G66zrdHHllK4goajhBygo1JqMHA38J6IRDZWUURuFJGlIrK0oKCgDUTVHCorcw1X1okVxg5S78ofUTnr8VX6sad1QERwdmnoZC506FAizr4AvFVEje5N7N1/I3ToMAK1Xmr96bDoJYhIhVPvNyJy1e1j2PgllJrWSUoZ00fdxkN0J3wqHltahza7bs2Rg91cbHb22tf3kS0lGWUaF9kSjDWRusVmS6jhhM4dMYaKX5dTu34L1SOeR/n8ODqbaxix5lpUnDH96OzRA5TCu3s34WNOxtmlywHdncTfcAM9fppL8oMPYA0PMzbUPfQgIQMHGnKldgIluIsD1O3q9GzbhjgcWMJC99PyodPmSkFEbMCFwMy6NKWUWylVZB4vA7YCjYalUkq9opQappQalmBqd82RycaCTACiyg2l4Fk5F29pDSgjMEyT9L8ErE449UEQIXSY4WO+JtrckDb4Khh8NVjssOZDw33BjCuN0JhgnPtqITIVrvsSr4rHnpLaWpepOYKpGyk4G4lyZ09O2VPOXCgPGdCf8HHjSH3maQBqN26gZq3x4lH2y3oAHJ06G5UGXAYn3wUxpllujz1m12FjWmayw97JtIgqtWOPM96TPVlZWGNigpsFW5r2GClMADYqpbLqEkQkQUSs5nFXoAdGaE7NUcz2MiO+gbPCDYA3M8OwIgLsHfbz5p7YB+7Pgh6nG2WTk7F36ED1zkq49gu8Xa+gctk6Y643cx7sWEBFtgPvTvMnU5FDTbGdzCe/wG+NwZuXH5xb1hxfhA4dir1TR0JO2Deegj1lz2/CaioFS2go6f99iYhx47DGxVHx3fcEyssBKP/WcFVdN31EfHeY8EjQ7bejY0fE4cCWmhLc1He42LoZPr4CPguODuaow+9vtfUEaF2T1PeBBUAvEckSkSlm1uXsu8B8CrDaNFH9CLipLkyn5uglpyob/GEEqgzrDU+lFXeFYfDmSD/AdI6t4Way0OHDqVq8BJU+irx/v0DWLVNR6aMheyWBdV+QNS+W4p+2GoXLs6nOd1CzIZOqn39C1dY2eABojh9Chwyh+zffYI3aNyCNrf5IoZFZB1evntQsWxbMV9XVSGgotsTGZyjEaiVy4kRiLru8xd7ibel7PAHYkxODwZOsMdEt0n6jfbZWw0qpK5pIn9xI2ixgVmvJomkfStx5RHqj8LsNy2NvpY2a8lisCbHYUlIOULsh4eNPpeyTT6j69Veqfv4Z5fXitnTHpfx4F8wCFYuvzLTQKM/GX2u875R/9x3AQfenOfaxJyUaMR0AW+y+foScPXtRNX8BEhpK1PnnUfTqazg6d9rvAz/18X+0qIy2uDhjLUGBNTYea0wMvtzcVnXYqHc0a1qNqkA+XWvMN5vIUDxVVqrzrYQOHnLQb1LhJ5+MhISQ9/gTBKoN3zO1pXawOvCWG2EK/ZVuwy1BRQ6+WmOaquonI9i6XSsFzV6I3Y4tPh5rbGyjsTOcvY3F6ZC+fYN7aJx1U0dthNhs2CKM/yFbQhLW6GgArDGt4wwPtFLQtBI+vx+/pZjuZlz2kIH9ISD4yryEDj14L6SWkBDCx4zBs307ltBQxOXCvWUrdBiBp9L4h/bVWqAyF8qz8XkNM9M6BaLXFDSNYUtNaXI3tsu0WHINGGBYA1ksOLq2/e53W1w0ANak9OC00VG5pqA5vllfsAux+Oleadj8hY4cHcwLOUTX1BGnGwvPYWNPwdmrJ7UbN0Gvs/HUGq6M/bUWKM82lIJ7z54EsduxxsUd6qVojmHif/974m++qdE8Z/fuRF1wAVHnTcIWG0unt98i9pqr21hCsHXuA4A1MbXeSCG69fprtZY1xzWrc40NNumVxqab0GFGwHgJCQn6fDlYwk8dh6tvX2IuuYTyb76l/OuvUSNfxxO7ApiPz21BlWUhFTn4agRH1654tm3Dlpy831jLmuOXiPH7cZNht5P6j8eC56HDhjVZtjWxJRlmtbbYmOBagl5T0Bx1bC40NpIllFWBYHiEtNkIGTjwkCNyWcPD6fLxLMJGj8bVpzeBsjJ8eXl4s8wYzUoIZGegijLxV/sIHzMGRPTUkeaoxpZomKJaY2PrjRRaTynokYKmVdhSYiiF8NJKasMdiMNB7FVXHfLU0d7U7VCtWbcOz+7d2FKS8eXk4lv1FdaKalCRODp3ImToEFz9tWtszdFL5Omn4y8qxpaYGHRtoZWC5qjC7fOzPn8H9vBIVHlOcKibdP99LdaHq2dPsNspff998HoJHTyY8pyv8G9fi7KbroXj4uj0zjuttvNTo2kLnD16kPxnI95CyJAhuAYMwJGe3mr96ekjTYvz5ZocPJSQFhKHr8aCLb7lzecsYWHEXHE5VfMXABAyaDAAPrcFX6jhi8YWn6AVguaYIuSEfnT5YCaWsLBW60MrBU2LMX/HJu78bDrP/5CB01VOt9BIQykktc6cfvzNN2OJNPzBhAweBIDfbcEXZviLqXNyptFomo9WCpoW44n5r/B90T/JKq7E6igjzS343Vbs9bbqtyS2mBgS770HV//+QZtyX60Fn81wfmfTZqgazUGjlYKmxSj3FiMWP5/c1QVPoIbO2ZUAOE9omcXlxoi55BK6fPiBEWUrIhS/NQm/LwxLaGjQ/bFGo2k+bR2j+RER2V0vFvPEenn3i0iGiGwSkTNbSy5N61HtN7xJLsgx5vmTs0oBcPRo1At6i2NNSMaXMhZfcSlWPXWk0RwSbRqj2eRfSqmn6ieISF8M76n9gFTgexHpqVRdcFTN0YA7UA4WWJBtKIXo7HLEKjg6dmyT/q2xMfiLisBiwRavY21oNIdCu8RoboTzgBlmsJ3tQAYworVk07QOfqkAYEX+CgBceVU4EsMahB9sTWyxcfiKi/EVFur1BI3mEGmPNYVbRWS1Ob1UtwMjDdhVr0yWmaY5SvD4fCiL4XzOG/BiEQsU+nF2aLs3dmtcLJ6dO/Fs24ajW+ssbms0xzptrRReAroBgzDiMj99sA3oGM1HJrvKChFRiPmTSiUSX7UNZ9fObSaDLTYOvF5cffoQf+ONbdavRnMs0aZKQSmVp5TyK6UCwKvsmSLaDdTfotfBTGusDR2j+Qhke0keAGnmxrF+pYYPeEevvm0mQ8iQwbj69aPDiy9gCQlps341mmOJZisFETls+z4RqR/p5AKgzjLpM+ByEXGKSBeMGM2LD7c/TduRVWaM2npHDwSge34AAGf/4W0mQ/hJJ9Fl1kc6oI5GcxgcUCmIyGgRWQ9sNM8Hish/mlGvsRjNT4rIGhFZDZwK3AWglFoHfACsB74GpmrLo6OLnMoiAAYnGnsSUgs8IApHr8HtKZZGozlImmOS+i/gTIy3eZRSq0TklANVaiJG8+v7Kf934O/NkEdzBJJfXQjA2C2fsrjDWDqWLcUWKojD0c6SaTSag6FZ00dKqV17Jem3eE0DCqsN6+O0tR/wwpgniC2rCcaW1Wg0Rw/NUQq7RGQ0oETELiL3ABtaWS7NUUaZp4RQvxhDz+Jt+MtrsUZHtLdYGo3mIGmOUrgJmIqxb2A3hjnp1FaUSXMUUuEtJSpguqnOWYWvWmGLa3mX2RqNpnXZ75qCiFiBZ5VSV7WRPJqjlGp/Gb1q/Xgqrdg3f4+v1oItSVsBaTRHG/sdKZgWQJ1ERK8WavaLJ1DOGQv9ZH4Xj3/jXFCCLaVTe4ul0WgOkuZYH20DfhWRz4CqukSl1DOtJpXmqMMnlSSU+vG7rdRmVwMh2NK7t7dYGo3mIGmOUthqfiyAXjnU7IPX5wNLFRHmK0N1gTGwtKV1bj+hNBrNIXFApaCU+j8AEQk3zytbWyjN0cX8HdtBFCGGPzyq802loN2QaDRHHc3Z0XyCiKwA1gHrRGSZiPRrfdE0RwO1Xj8PfP0RAK5qw/qotthQCtZ4HehGoznaaM700SvA3UqpHwFEZByGM7vRrSeW5khnd1kxEz+8DHfeuUjUcjqqKPAari5UQBC7BUtYWDtLqdFoDpbm7FMIq1MIAEqpuYD+bz/OWbp7KwF7LpEdPscVuZVzbJ2NDDFGC7bYaMQ81mg0Rw/NUQrbROTPItLZ/DyEYZGkOY7JrSgFoIZcfMrLaHc0AM6ehsWRLaVtQnBqNJqWpTlK4XdAAvAxMAuIN9P2ixlZLV9E1tZL+6eIbDQjr30iItFmemcRqRGRlebnv4d0NZo2o6CqDIAYZxyxrli6lRvusMJGjATAlqDXEzSao5EDKgWlVIlS6nal1BCl1FCl1J1KqZJmtD0NOGuvtO+AE5RSA4DNwP318rYqpQaZn5uaewGa9qGwuhyAp8f+i3cnvosqMhzihYww4ibpRWaN5uikOdZH39W90ZvnMSLyzYHqKaV+Bor3SvtWKeUzTxdiRFjTHIUU1xhKoUt0OukR6fiKjfeE0KFDQQR7UlJ7iqfRaA6R5kwfxSulSutOzFFCYgv0/Tvgq3rnXURkhYj8JCJjmqqkYzQfGZS5je0qYVYj7KWvtBJriAVbbCzpr7xM9GWXtad4Go3mEGmOUgiISHDVUEQ6AepwOhWRBwEfMN1MygE6KqUGA3cD74lIZGN1dYzmIwNfTS5WpXCtfM84L6vGFmEHIHzMGGwxMe0pnkajOUSaoxQeBH4RkXdE5F3gZxquBRwUIjIZOAe4SimlAJRSbqVUkXm8DMOtRs9D7UPT+vh9pYQFAsiaDwHwVXiwRYa0s1QajeZwaY6bi69FZAhwIsYI4U6lVOGhdCYiZwF/BMYqparrpScAxUopv4h0BXqgzV6PaPzeCq75WeFJXYqjdCf+Kj/OjuHtLZZGozlMmhwpiEgnEYkCMJVAFXAGcE1zXGmLyPvAAqCXiGSJyBTgBQynet/tZXp6CrBaRFYCHwE3KaWKG2tX0/74/AESSss4ZQVk/RpLYOm7+GoFa0xUe4um0WgOk/2NFD4ALgDKRGQQ8CHwD2Ag8B/g+v01rJS6opHk15soOwtjD4TmKKC42oPT5wHAXWon86E3UH67doCn0RwD7E8phCilss3j3wJvKKWeFhELsLLVJdMcsRRWeHB6DaUQNrg3nl1ZRI3pSuTke9tZMo1Gc7jsTynUd1wzHnNxWSkV0D5tjm8KK904fMZ2k8SH/4Grd+92lkij0bQU+1MKP4jIBxjmojHADwAikgJ42kA2zRFKUZUbh9dwa2EJ13GXNJpjif0phTuBy4AU4GSllNdMT8YwU9UcpxRWeLB7AwBYw7XDXI3mWKJJpWDuIZjRSPqKVpVIc8STX1GNw3xF0DETNJpji+YE2dEcx2SXl3DHV/8i3XomDjGmihbuyOIstyJgBbHb21lCjUbTkmiloNkvT/z8IRtrP2GzewXOwpsRHASslYR4IOBszoZ4jUZzNHFQ/9Wmh9QBrSWM5shCKcWvuxciyo5y7mD8mJ9Z9MAEpt84iBAP4LS2t4gajaaFaY7r7LkiEikiscBy4FUReab1RdO0N4u3F1Nj3UzvqJHcPPBm/rftf/yc9TNV3ipC3IBLDzQ1mmON5vxXRymlykXkeuBtpdTDIrK6tQXTtA+3ffE8ecWhxMhANhZuxxJbxjk9x3BF70v4OvNr/r7w7/xh2B9wecAS4mxvcTUaTQvTnOkjm7k34VLgi1aWR9OObMjP4sfC19jk+YAdRdUEnFsBODntROxWO/ePvJ/sqmw+z5hNiEdhDXG1s8QajaalaY5SeBT4BshQSi0xvZhuaU7jTcRpjjWjuW0x/8aY6SIiz4lIhhnDecihXJDm0Hlq/ruIBAjYs3l1ShdO6l9KnCuWLqs/htoyhicNJ8IewcLcxYS4wabNUTWaY47mxGj+UCk1QCl1i3m+TSl1UTPbn8a+cZrvA+YopXoAc8xzgLMxXGb3AG4EXmpmH5oWwOf3s7Toa5wqBYCZG2cyZ8ccxoSkUTPjCdSH12EFBicNptbvJsQDjjC9m1mjOdZozkLzk+ZCs11E5ohIgYj8tjmNNxanGTgPeMs8fgs4v17628pgIRBtTltp2oA3ln9LwFbEBZ0n0yOmB2+tfwu/8jNlvZsdP8RT/sN8mPMoQ5OGAhhKIaLR4HgajeYopjnTR2copcoxoqVlAt2Bw3GHmaSUyjGPc4G6CO9pwK565bLMNE0b8P6GD8Afyu2jLmB8+ngArupzFWFrNwNQvKsT6tfnGWqPRQIKlxeskdHtKLFGo2kNmrXQbP79DfChUqqspTo3XWkcVLxnEblRRJaKyNKCgoKWEuW4ZlNBNgWB5fQKH0+EM4SLe17MBd0v4IYu51OTVQlAbVY5NeVR9F3wKjE+YxezRSsFjeaYozlK4QsR2QgMBeaYoTNrD6PPvLppIfNvvpm+G0ivV66DmdYApdQrSqlhSqlhCTqoS4vw9ILpiAS4ddhVACSHJfPoSY8SkbOK2mIHkaeeiCUykqLcftgzf2GoxzBFtUTGtKfYGo2mFWjOQvN9wGhgmOkptRpj/v9Q+Qy41jy+Fvi0Xvo1phXSiUBZvWkmTSuyuOBrQgM9GNf1hAbp3hVz8HsshJ5yOnHXTaZy+VYqC6K4c8dOAKzRce0hrkajaUWas9AcCtzCHmugVGBYcxpvIk7z48DpIrIFmGCeA3wJbAMygFfNPjWtzOrcTPy2fIYlnNIwo2w3tfO/BcA1YCCxU6bg6NyZ3OWxxFUYLlItUfFtLa5Go2llmrOj+U1gGcZoAYwpnQ9pxka2JuI0A5zWSFkFTG2GPJoWZPaGXwA4q9voPYmeaphxBTV5AcRhx9WzJ2K3k/TgA+y64UbKd4YAYInQJqkazbFGc9YUuimlngS8AEqpahqG6tQcxSzOWYoKODmzR729gr/8C3JWUe3tjqtP36B77NCRI8FmpbrAXFMID28PkTUaTSvSnJGCR0RCMK2ERKQb4G5VqTRtxu6a9URZe+CwmT+F8hxY8ALVYROo3bKepPuvDJa1OBw4u/fAvXGjca53NGsOEq/XS1ZWFrW1h2OromkuLpeLDh06YD+IuCfNUQoPA18D6SIyHTgJmHxIEmqOKDKL8/HZcugdNT6Y5vvi/1AVfgp3O7HGxRF96aUN6rj69NFKQXPIZGVlERERQefOnRHREw6tiVKKoqIisrKy6NKlS7PrHVApKKW+E5HlwIkY00Z3KKUKD11UzZHCx+uN9YTTOgwCpaB0Bzv+NQdPRRywgsR778ESEtKgjqtPb8o+MY719JHmYKmtrdUKoY0QEeLi4jjY/VzNdYjvAkrM8n1FpM6FheYoZt7OlaCEs2fdg3fhsxCaiKfCRvjYk7B37ErMFfvaCbj69DEO7HYsDkfbCqw5JtAKoe04lHt9QKUgIk8AlwHrgICZrACtFI5iPL4AGaVbSbRGkPdZIZXJG4jssAKIIeGue3D17t1oPaepFKyhoW0orUbTMhQVFXHaaYbxY25uLlarlbpNsIsXL8ZxkC86Gzdu5LrrrmP58uX8/e9/55577mlxmdua5owUzgd6KaX04vIxxC8ZBfhtufxmixvlt1CVF4Y4w7BEhuHs2bPJetbwcOwdO4Lf34bSajQtQ1xcHCtXrgTgkUceITw8/LAe5LGxsTz33HPMnj27ZQQ8AmiOSeo2oPlL15ojiu3FhWzJqyAjv7LBZ+aSTKyOQoasrUDsFpQvQMW2AKHDhiOW/f8swkaOwN4xfb9lNJqjhTlz5jB48GD69+/P7373O9xu4/23c+fO/PGPf6R///6MGDGCjIyMfeomJiYyfPjwg7LuOdJpzkihGlgpInOoZ4qqlLq91aTStAjrc3O59KvfUJtzEb7ygQ3yLI584tP8JO4QYs6bQMWyjXh37iR0+PADtpv8l78YC9MazWHwf5+vY312eYu22Tc1kofP7dfs8rW1tUyePJk5c+bQs2dPrrnmGl566SXuvPNOAKKiolizZg1vv/02d955J198cewHn2yOUvjM/NRHPxGOAl5btAixeBjbv4rzOg5ukLe+bB47PldIQIi89FokeR5FL/23WUpBjqG3Is3xjd/vp0uXLvQ0p0yvvfZaXnzxxaBSuMI0trjiiiu466672kvMNqU5SiFaKfVs/QQRuaOV5NG0ENUeH99tWQcJIM48Jg1MbZCfs6oUChRYDN9Gjh49cXbtiqtf3/YRWHPccTBv9O1Ffeud48VqqjlK4Vrg2b3SJjeSpmkHXl78Dct2FJBobzg9lFNai5s8nMC20m371NtatpXBpYIj2oZYrVjDw4k699w2klqjOTKwWq1kZmaSkZFB9+7deeeddxg7dmwwf+bMmdx3333MnDmTUaNGtaOkbUeTSkFErgCuBLqISP3powj2DbHZbESkFzCzXlJX4C9ANHADULfT4gGl1JeH2s/xwmtr/kt1oARX7oP75KV2rKIIKKgpoMxdRpQzCgCP30NGaQYTSwLY43VMBM3xi8vl4s033+SSSy7B5/MxfPhwbrrppmB+SUkJAwYMwOl08v777+9TPzc3l2HDhlFeXo7FYuHf//4369evJzLy6A1Vu7+RwnwgB4gHnq6XXgGsPtQOlVKbgEEAImLF8Lr6CXAd8C+l1FOH2vbxiDtQicVRxHf3DiXWFdsg74ov3qC8xIY34GNb2TYGJw7mi21f8Odf/oxP+YgpVdh760BFmuOTRx55JHi8YsWKRsvce++9PPHEE022kZycTFZWVkuL1q40aXuolNqhlJqrlBqllPqp3me5UsrXQv2fBmxVSu1oofaOK6o9PjoVlNIlV7G6YF89vbN8ByOrjHCaW0u3AvBN5jfEhsTyZM+bcbgFR3rHNpVZo9Ec2TSpFETkF/NvhYiU1/tUiEhL2ZFdDtQfk90qIqtF5A0RaXReQ8do3kNWSQ2T51bx+6/8rMpe2CCvtLaUcm8FJ1bXEBJQbC1YS0AFWJG/gpNST+LUcsOnkb1Lj/YQXaM54snMzCQ+/vgLJLW/XUpXASilIpRSkfU+EUqpw54wExEHMAkjYA8Ykd26YUwt5dBwyiqIjtG8h8yiUsKrFalFsGpLQ/vpnRVGyMxOPj9dvF627fyJraVbKXOXMTRpKJ6thqdTe4/+bS63RqM5ctmfUvik7kBEZrVC32cDy5VSeQBKqTyllF8pFcAIxzmiFfo8psgoKiS8FlxeyCouxpe9MphXpxQ6eNPoHprOxpo8Fm34AIAhSUPw7swEwNFz4N7NajSa45j9KYX6RrldW6HvK6g3dSQiKfXyLgDWtkKfxxSZhXmE1xjHMaXClo2zg3m7yncSVxbAO72K80tGUWy18tzm90kMSaBDeAe82blYnGCNjm4X2TUazZHJ/pSCauL4sBGRMOB04ON6yU+KyBoRWQ2cChwf2wcPg7KiHThMv3QpxbA2b1kwb3vhOkbtCEAAUne6OTd5NDUiDMWJiOApKMMR42wnyTUazZHK/pTCwLqFZWBASy40K6WqlFJxSqmyemlXK6X6K6UGKKUmKaVyDqeP4wFf4R6jrc7FsNGcMgLYWLSeQbsMjVG7di1/GvckAwjh7LydEPDjLa7BnhDV5jJrNO1JUVERgwYNYtCgQSQnJ5OWlhY893g8h9TmuHHj6NWrV7Cd/Pz8fcpMmzYNEeH7778Pps2ePRsR4aOPPjrk62kNmtynoJSytqUgmoPHUrZHb3Yrc/C+qoKaUqptDjJrC+mSY8wAerZvJ9xjYfqQP8KsKXjevglPhRDRrVd7ia7RtAst7Tq7junTpzNs2LD9lunfvz8zZsxgwoQJALz//vsMHHjkrek1x3W25gikotZLSK2xsVwiwkgusbDZYceXvZzNJZtxeBThhRZcJ5wAQO269dBrIsoeTu5b32OxWoi57eH2vASN5ojgcFxnHwxjxoxh8eLFeL1eKisrycjIYNCgQcH8ZcuWMXbsWIYOHcqZZ55JTo7x0vfqq68yfPhwBg4cyEUXXUR1dTUAkydP5vbbb2f06NF07dq1xUYczQ3HqTkCKKyq5K8/vk9n5ymU1fiI9FYBEHJCHwKLl+NF2J45l40pPemerRAFsddcTfYf/0Tt2jWEnTiSkpIhVOVkkHTTZdjT0tr5ijTHNV/dB7lrWrbN5P5w9uPNLt5SrrOvu+46rFYrF110EQ899FCjzvNEhAkTJvDNN99QVlbGpEmT2L59OwBer5fbbruNTz/9lISEBGbOnMmDDz7IG2+8wYUXXsgNN9wAwEMPPcTrr7/ObbfdBkBOTg6//PILGzduZNKkSVx88cUHc7caRY8UjiL+veBDfih6jhfmf8+0+ZlEeA3To9AhwxB/gIRS2Ji3nI27FzBwVwBECB83DntaGlXzF5D94IPkfZFB+LA+xNz2UPtejEZzBNCY6+yff94Tabi+6+wFCxY02sb06dNZs2YN8+bNY968ebzzzjtN9nf55ZczY8YMZsyYEWwbYNOmTaxdu5bTTz+dQYMG8be//S3oPmPt2rWMGTOG/v37M336dNatWxesd/7552OxWOjbty95eXmHfiPqoUcKRxGbi43h618vjeOK3hN5//a/A4ZSAOhaJKx3bGaDVHJlbgBHekeskZG4+ven4uuvwWIh7sYbSbjjdsSql4w07cxBvNG3F3u7zvb7/QwdOhSASZMm8eijj5JmjrgjIiK48sorWbx4Mddcc02j7Y0YMYI1a9YQGhoaVEQASin69evXqOKZPHkys2fPZuDAgUybNo25c+cG85xOZ4M2WgKtFI4icqu3gcDW0s2ICKrKjc8KIUOHgt3OyPxwPujso8BdSFKZBXtfY3tJ7DXXYE9KIuaKy3F07ty+F6HRHEEcrOtsq9UaXKgG8Pl8lJaWEh8fj9fr5YsvvgguJDfF448/jsvlapDWq1cvCgoKWLBgAaNGjcLr9bJ582b69etHRUUFKSkpeL1epk+fHlRCrYVWCkcRkUUZXD/Xz09XLoJRYKn2UusCi8tFyAknMKSgmA9tVXjxE1FmCTq7Cx0ymNAhgw/QukZz/HG4rrPdbjdnnnkmXq8Xv9/PhAkTgvP/TXH22Wfvk+ZwOPjoo4+4/fbbKSsrw+fzceedd9KvXz/++te/MnLkSBISEhg5ciQVFRWHf+H7QVpqyNEeDBs2TC1durS9xWgTiqor+MefTmTKdwH+ebmd1x5eyazz+pNaDCf9so78p5+m6M1p9Jz2AEUfT6Xo4ySSHrif2CaGsRpNe7Bhwwb69OnT3mI0i86dO7N06dKj3ileE/e8yTByeqH5KGH+jg2klBgKPK7IT3Z5Fo5ahS/EWBsIHTYMfD5qvZ2JGHgvAPYO6e0mr0ajOTrR00dHCctyNtLDjHeXVqTYnPElzlpFINYBQMjgwSBC9bLlODoZexMcHbVS0GgOlczMzPYWoV3QI4WjhM1Fm0ktNkYKqUWKDdu+JqwWCDMWrKyRkTh79aJ6yRI8Ow13F/YOHdpLXI1Gc5TSbkpBRDJNB3grRWSpmRYrIt+JyBbzrw4gbFJStoHEMkMppBcLP5dsILwGLBFhwTLhY06metkyalevwZaUhGUvCweNRqM5EO09UjhVKTVIKVXnNOQ+YI5SqgcwxzzXAK6SLCxKcHRMJbpCsdNvx+UFa9Qep3aREyeCz0flTz/hSNdTRxqN5uBpb6WwN+cBb5nHbwHnt58oRw4l1TVElRqxlsPHjwfg5DwjnKYjdo9lhLN3bxxdjb0J9o469rJGozl42lMpKOBbEVkmIjeaaUn1XGbnAkntI9qRxbzMTaSUGMcRp58FwG8txgab+G4nBsuJiDFaQC8yazRNkZeXx5VXXknXrl0ZOnQoo0aN4pNPPjlwxYNk48aNjBo1CqfTyVNPPdXi7bcW7akUTlZKDcEIyzlVRE6pn6mMDRT7bKIQkRtFZKmILC0oKGgjUduXxVkbSSlRKJcFV//+YLEQsWQTAIkp3RqUjTr3HMTpxHWCjr2s0eyNUorzzz+fU045hW3btrFs2TJmzJgR9DPUksTGxvLcc8+1iGvutqTdlIJSarf5Nx8jHvQIIK8uLKf5d59oFUqpV5RSw5RSwxISEtpS5HZjY0EGKcXgSIrA4nBgT++Ae0sGrn79CK3nehfA0akTPRctJPzkk9pHWI3mCOaHH37A4XA02LXcqVOnoNdRv9/Pvffey/DhwxkwYAAvv/wyAHPnzmXcuHFcfPHF9O7dm6uuuuqAvoYSExMZPnw4dru99S6oFWiXfQpmOE6LUqrCPD4DeBT4DLgWeNz8+2l7yHekUVS5ifRCRegQY0oo+aE/4y8tIXLixEYd22mrI83RwBOLn2Bj8cYWbbN3bG/+NOJPTeavW7eOIUOGNJn/+uuvExUVxZIlS3C73Zx00kmcccYZAKxYsYJ169aRmprKSSedxK+//srJJ5/covIfCbTX5rUk4BPTA6ENeE8p9bWILAE+EJEpwA7g0naS74jB4wsQUrmDqCpw9R8EGKanGo3m8Jk6dSq//PILDoeDJUuW8O2337J69epgwJqysjK2bNmCw+FgxIgRdDD3/gwaNIjMzEytFFoKpdQ2YJ84dEqpIuC0tpfoyCSvoozVu6pIKS4FwDXslP1X0GiOIvb3Rt9a9OvXj1mzZgXPX3zxRQoLC4OhNJVSPP/885x55pkN6s2dO7eBm2qr1YrP52sboduYI80kVWOycmcJE2aex+0/3kpagR8A5wDt6VSjORzGjx9PbW0tL730UjCtLrwlwJlnnslLL72E1+sFYPPmzVRVVbW5nO2J9n3Ugvz5+7fZnOMhznL4D+95W7dCpyJs9iK65il8sXas4eEtIKVGc/wiIsyePZu77rqLJ598koSEBMLCwnjiiScAuP7668nMzGTIkCEopUhISGD27Nn7bfMvf/kLw4YNY9KkSQ3Sc3NzGTZsGOXl5VgsFv7973+zfv16IiMjW+vyWgTtOruFUEox4I2xiD+MpMrD34gdErWJ7bbn6RSRzj1Pbie1Sxo93/++BSTVaNqPo8l19rHCwbrO1iOFFmJjXhGX/lpIjauIR184CZulebdWKcU/l/6Tc7qeQ9+4vsH0l1atJ+p+PycmCu5SiO2n9x1oNJrWR68ptBDfb17PaasUp64KsCN7yT75Ze4yJs2exIr8FQ3ScyuyeHft28xa8myD9B1bljB4m8K9cBsArqHHnpWDRqM58tBKoYXYsuknYqogtQg2LXljn/ztRRvZXradD9e82SB9W8aPvPyCH+v3ixuk+1euBiBq7GAcHVMJGb3/uK8ajUbTEujpoxbCssMYAViA3BULWH/yOpQF+sX1AyA/ayEAP+6eh9vvxmk1zNtyVs2lXxXEZrqp8dXw3Y7viHXFkpxZjd8GKc9PQxyOdrkmjUZz/KGVwiFQWFVJRY3CIsZAq7zGR0xxdjC/Mt/D1P9dQUJEBz646EsAirPW8NSrPl6eqPg161fGdzK8nZZv3QJAWqHiy3Xv8sjK5wD4R5ZCdYnTCkGj0bQpWikcJO+v+onHlt2Lp/wE3LkXBdPvKa/CZwVfqBNbkQdHcYCaoh343ZVYneHUbt5Kx0I4a4Wfb4bNDCqFwO5SADoUwR2rX8GGhaGVHjrnQdQZ49vjEjUazXGMXlM4CKYt+5G/L78TLF4cMUu559wwnrl0IE9fMoDYUj9VMVbcPTrQZ5fikel+bvw8QPbSVwDw5hcBMDxDMTd7IeWecvB5CC00dkW6vOAoquFPX9fw8DdhWBXEnKzXETSalqatXGdPmzYNEeH77/eYks+ePRsRCbrROBLRSqGZZORX8PTi/2AllOlnfUSMM5rllW9z3qAUxvZ2kVSkCCSHE9V/MMmlEF0FnQpg2/Jp4K1BStwAuGqETrv8fLbqDUp2LyaxCDzRhgO70RsUA1dYqdxaiyU0lJC9PKBqNJrDoy1dZwP079+fGTNmBM/ff/99Bg7cx8PPEUWbTx+JSDrwNoZTPAW8opR6VkQeAW4A6oIkPKCU+rK15Xlz2Xc8v+JJ/MXnYqntvU9+wLURFfErnoIzsXfcyNXRQxjgcnDr4Fv568K/cu3X19LfmcJvSsE9Ko0Ow8eS9dZH2Hp2h80Z5OSVwo9/x14puEMsuHxWJm7wMjP1FfrwJckl4J44EMf/FnHBogCI0H3O91hCQ7Ee4TsfNZqjjea4zr7vvvuYO3cubrebqVOn8vvf/565c+fyyCOPEB8fz9q1axk6dCjvvvsuplPPJhkzZgzz5s3D6/XidrvJyMhgUL2XvWXLlnH33XdTWVlJfHw806ZNIyUlhVdffZVXXnkFj8dD9+7deeeddwgNDWXy5MlERkaydOlScnNzefLJJ7n44otb9B61x5qCD/iDUmq5iEQAy0TkOzPvX0qpNgtR9Pe57zEj83HEGiA08W1Oifo3Lkt0gzLzKl+n0Lee0M6ZJBUoLvhkLu6fB3HJeX8i5OTHeHzRYxRlr2SSgtQTRhE+bhypTz+Fs0sXtl94EZU1sTD/ecIqUqlKiiSu8yCGLVvEu8M9vF+9gynKQuyJ4/Gt2IkrO4fQ4cOwJye31S3QaNqN3Mcew72hZV1nO/v0JvmBB5rMb2vX2SLChAkT+OabbygrK2PSpEls374dAK/Xy2233cann35KQkICM2fO5MEHH+SNN97gwgsv5IYbbgDgoYce4vXXXw8qrpycHH755Rc2btzIpEmTjn6lYIbbzDGPK0RkA5DW1nL4/H5mbn2ZEQXwh3d9/HtSDbVj3+bJM18JlimuLebTDzbSPbo7GaUZ3Pm1F8+uULZtCSUp91nOmXofwz7Jo3yzgwBCzKjfIDYbUb/5DcrrxWcTpDyEmkghphwCPRNIvOcPVP/2av7+oeL73gFAkdR/OIXdf8GXnUPEWWc2LbRGo2lR2sJ19uWXX85zzz1HWVkZTz/9NI899hgAmzZtYu3atZx++umAMUpJSUkBYO3atTz00EOUlpZSWVnZwGvr+eefj8VioW/fvuTl5bXo/YB2tj4Skc7AYGARcBJwq4hcAyzFGE2UtFbf76z8AWUr5uYfahCfnSnzfNzQcz5FVQX8uPsnZmfMZlz6OGweP/9IuJ7Mrf+h465txE2+Evf2XeT9PI+aR5+ifEco4aeMJvryq3D23jP9JHY7lR1iiNxdRv65vyX+wzlUpqbg7N6djq+9hkyZwoXza8FiwdWlK65evan65VciJpzeWpes0RxR7O+NvrVoD9fZI0aMYM2aNYSGhtKzZ89gulKKfv36sWDBgn3qTJ48mdmzZzNw4ECmTZvG3Llzg3n15WgN33XtttAsIuHALOBOpVQ58BLQDRiEMZJ4uol6LRKj+f0NHzNgpyJspx1Xn95EFlg4IVMx74cHeH3lf1lVsIpnlz/L3d+6UL+7h85vZGANsxN/+x9Ie+ZfODqmUb4jlMgzTqXDy68RMX78PvOLgW6dSM/1sz7tJBw+cKUZbxkhJ/Sj25zvSfrzQyTdfz8Wp5PY311Hp3fexp6UeMjXpNFo9k97uc5+/PHHgyOEOnr16kVBQUFQKXi9XtatWwdARUUFKSkpeL1epk+fftj9HwztFY7TjqEQpiulPgZQSuXVy38V+KKxukqpV4BXwPCSeij951WWketdyINzPVijIug4bRpbf3MOFy8q5tUO87CU2Lhrs5vv0xwMWV1FeGotHk8UsTffjSU0FID0196k7NPPiLt+SpOLTeF9T8AxZwUL5n1OZyAivVswzxoeTuxVVwXPbTEx2IYOPZTL0Wg0zaQtXWfX5+yzz94nzeFw8NFHH3H77bdTVlaGz+fjzjvvpF+/fvz1r39l5MiRJCQkMHLkSCoqKg7rug+GNnedLcYT9C2gWCl1Z730FHO9ARG5CxiplLp8f20dquvsGat/5rPZt/Dn9/wk3nMncdf/nsJXX6Xg6Wf403VWbv4yQOc8475Yo0Pp/vjVWMbcCtaD06EVixeSdc11LOkhDN+iSPjwHeL7DztoeTWaYwXtOrvtORpcZ58EXA2sEZGVZtoDwBUiMgjDTDUT+H1rCXB5aion/liNJzySmCuvBiDmssvIe+k/3DG7ltQSiL32Gmo3bSZ28rVYxo07pH7Chw6ntlMiw7fkAxDdqecBamg0Gk370h7WR7/QuJZq9T0JddRkZFKT4yThlquD00HWyEhiLrsUy5tvo1ISSLznHsRuP6x+xGql8wOPkPv7W6h2WbDpfQcajeYI57jc0WzvNZT4W24m5robG6QnTP4dlshIUu+4+7AVQh0xY0/Fc+IApEfnFmlPo9FoWpPj0iGeLSaGhNtv3yfdnpREz4ULEEvL6soBr77bKqZjGs3RiFLqgDuBNS3DoTx3jsuRwv5oaYUAxp4Fi3aBrdHgcrkoKirSL0ltgFKKoqIiXC7XQdU7LkcKGo2mfejQoQNZWVkczh4jTfNxuVzBXdjNRSsFjUbTZtjtdrp06dLeYmj2g54+0mg0Gk0QrRQ0Go1GE0QrBY1Go9EEaXM3Fy2JiBQAOw6hajxQ2MLitARaroPnSJVNy3VwHKlywZEr2+HIVaiUOquxjKNaKRwqIrJUKXXEOSHSch08R6psWq6D40iVC45c2VpLLj19pNFoNJogWiloNBqNJsjxqhReOXCRdkHLdfAcqbJpuQ6OI1UuOHJlaxW5jss1BY1Go9E0zvE6UtBoNBpNIxx3SkFEzhKRTSKSISL3taMc6SLyo4isF5F1InKHmf6IiOwWkZXmZ2I7yJYpImvM/peaabEi8p2IbDH/xrSxTL3q3ZOVIlIuIne21/0SkTdEJF9E1tZLa/QeicFz5m9utYgMaWO5/ikiG82+PxGRaDO9s4jU1Lt3/21juZr87kTkfvN+bRKRM9tYrpn1ZMqsCwbWxverqedD6//GlFLHzQewAluBroADWAX0bSdZUoAh5nEEsBnoCzwC3NPO9ykTiN8r7UngPvP4PuCJdv4ec4FO7XW/gFOAIcDaA90jYCLwFUZwqROBRW0s1xmAzTx+op5cneuXa4f71eh3Z/4frAKcQBfzf9baVnLtlf808Jd2uF9NPR9a/Td2vI0URgAZSqltSikPMAM4rz0EUUrlKKWWm8cVwAYgrT1kaSbnYcTWxvx7fvuJwmnAVqXUoWxcbBGUUj8DxXslN3WPzgPeVgYLgWgRSWkruZRS3yqlfObpQuDg3Ga2klz74TxghlLKrZTaDmRg/O+2qVxmPPlLgfdbo+/9sZ/nQ6v/xo43pZAG7Kp3nsUR8CAWkc7AYGCRmXSrOQR8o62naUwU8K2ILBORuvB0SUqpHPM4F0hqB7nquJyG/6jtfb/qaOoeHUm/u99hvFHW0UVEVojITyIyph3kaey7O1Lu1xggTym1pV5am9+vvZ4Prf4bO96UwhGHiIQDs4A7lVLlwEtAN2AQkIMxfG1rTlZKDQHOBqaKyCn1M5UxXm0XszURcQCTgA/NpCPhfu1De96jphCRBwEfMN1MygE6KqUGA3cD74lIWwYSPyK/u3pcQcOXjza/X408H4K01m/seFMKu4H0eucdzLR2QUTsGF/4dKXUxwBKqTyllF8pFQBepZWGzftDKbXb/JsPfGLKkFc3HDX/5re1XCZnA8uVUnmmjO1+v+rR1D1q99+diEwGzgGuMh8mmNMzRebxMoy5+55tJdN+vrsj4X7ZgAuBmXVpbX2/Gns+0Aa/seNNKSwBeohIF/ON83Lgs/YQxJyvfB3YoJR6pl56/XnAC4C1e9dtZbnCRCSi7hhjkXItxn261ix2LfBpW8pVjwZvb+19v/aiqXv0GXCNaSFyIlBWbwqg1RGRs4A/ApOUUtX10hNExGoedwV6ANvaUK6mvrvPgMtFxCkiXUy5FreVXCYTgI1Kqay6hLa8X009H2iL31hbrKQfSR+MVfrNGFr+wXaU42SMod9qYKX5mQi8A6wx0z8DUtpYrq4Ylh+rgHV19wiIA+YAW4Dvgdh2uGdhQBEQVS+tXe4XhmLKAbwY87dTmrpHGBYhL5q/uTXAsDaWKwNjvrnud/Zfs+xF5ne8ElgOnNvGcjX53QEPmvdrE3B2W8plpk8DbtqrbFver6aeD63+G9M7mjUajUYT5HibPtJoNBrNftBKQaPRaDRBtFLQaDQaTRCtFDQajUYTRCsFjUaj0QTRSkHT5oiIEpGn653fIyKPtFDb00Tk4pZo6wD9XCIiG0Tkx0byeojIFyKy1XQV8uPeu8LbEhE5X0T61jt/VEQmtJc8miMbrRQ07YEbuFBE4ttbkPqYu1ibyxTgBqXUqXu14QL+B7yilOqmlBoK3Iax/6PVqNtU1QTnY3jYBEAp9Rel1PetKY/m6EUrBU174MMIJXjX3hl7v+mLSKX5d5zphOxTEdkmIo+LyFUisliM2A/d6jUzQUSWishmETnHrG8VI67AEtMB2+/rtTtPRD4D1jcizxVm+2tF5Akz7S8Ym4teF5F/7lXlKmCBUiq4U14ptVYpNc2sG2Y6f1tsOlY7z0yfLCIfi8jXYvjKf7KeDGeIyAIRWS4iH5r+cOriXjwhIsuBS0TkBvP6VonILBEJFZHRGL6i/ilGDIBu9e+xiJxmyrHGlMtZr+3/M/tcIyK9zfSxsieewIq63e+aYwetFDTtxYvAVSISdRB1BgI3AX2Aq4GeSqkRwGsYb+N1dMbwo/Mb4L/m2/sUjK3/w4HhwA2mCwUw/OnfoZRq4MdGRFIx4g+Mx3DaNlxEzldKPQosxfAjdO9eMvbD2O3aFA8CP5hyn4rxsA4z8wYBlwH9gcvECLQSDzwETFCGk8KlGM7Y6ihSSg1RSs0APlZKDVdKDcRwtTxFKTUfY7fwvUqpQUqprfWuz4Wxc/cypVR/wAbcXK/tQrPPl4B7zLR7gKlKqUEYXkRr9nOtmqMQrRQ07YIyPD6+Ddx+ENWWKMPPvBtjO/+3ZvoaDEVQxwdKqYAyXB5vA3pj+HC6RowoWosw3AX0MMsvVobf/r0ZDsxVShUoIx7BdIygLM1GjEhna0WkzqHZGcB9phxzARfQ0cybo5QqU0rVYoxaOmEETOkL/GrWudZMr2NmveMTzFHPGowRS78DiNcL2K6U2myev7XX9dXJvIw99/dX4BkRuR2IVnviNGiOEQ5mDlWjaWn+jfFW/Wa9NB/my4qIWDAi5NXhrnccqHceoOFveW/fLQrDN8xtSqlv6meIyDig6lCEb4J11HuwKqUuEJFhwFN1XQIXKaU27SXHSBpenx/jmgT4Til1RRP91Zd9GnC+UmqVGF5Rxx36ZUA9eepkQSn1uIj8D8MPz68icqZSauNh9qM5gtAjBU27oZQqBj7AmNqpIxMYah5PAuyH0PQlImIx1xm6YjhV+wa4WQx3xIhIz3rTNk2xGBgrIvHmQu4VwE8HqPMecJKITKqXFlrv+BvgNhERU47BB2hvodled7N8mIg05a45Asgxr/GqeukVZt7ebAI617WNMSW33+sTkW5KqTVKqScwvA73PoD8mqMMrRQ07c3TQH0rpFcxHsSrgFEc2lv8TowH+lcYni5rMdYd1gPLxQjS/jIHGCkrw/XwfcCPGF5jlyml9usyXClVgxG34CZzQXwBxprA38wif8VQdKtFZJ15vr/2CoDJwPsishpYQNMP4j9jTI39CtR/e58B3GsuDAcX5M37ch3woTnlFAAOFIz+TnM6bDWGZ9GvDlBec5ShvaRqNBqNJogeKWg0Go0miFYKGo1GowmilYJGo9FogmiloNFoNJogWiloNBqNJohWChqNRqMJopWCRqPRaIJopaDRaDSaIP8PHHDSBONxs/QAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAADgCAYAAADsbXoVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABYq0lEQVR4nO3dd3hURdvA4d+zm95JIwkQunQIEJpIUVBUFLG3V0R9VWyIir1jw/opFnxRFEGkCIqIIE1RQQFpCgSQDglJSEJ63TLfH+dkTUghgRQIc1/XXtmd0+bswj57ZuY8I0opNE3TNA3AUt8V0DRN004fOihomqZpLjooaJqmaS46KGiapmkuOihomqZpLjooaJqmaS46KGgaICKDRSS+vuuhafVNBwWtTonITSKyQURyRCRRRJaIyHmnsD8lIm1KvB4sIk5z/9kisktEbquZ2ldYh2ki8nIFy64QkS0ikiUiqSLyk4i0FJGPzTrmiEiRiNhKvF4iIi3Mc9t83P5CzfUPVFIfJSK55r4SROQdEbGWWH6DiKwz1zlqPr9XRKTE+RSZ2x8TkeUi0t5c9sJxdc0RkYyaeB+104MOClqdEZGHgXeBV4HGQDTwEXDFSezLrZLFR5RSfkAA8DjwiYh0rHaFT5EZrKYDjwCBQEvgQ8ChlBqjlPIz6/kqMKf4tVLqkhK78RGRziVe3wTsr8Lhu5n7HmJuc6dZp0eA94A3gQiMz2EM0B/wKLH9G+b2TYGjwLQSy0rW1U8pFVSF+mhnCB0UtDohIoHABOA+pdQ3SqlcpZRNKfW9UupRc53eIvKHiGSYVxEfiIhHiX0oEblPRHYDu0XkV3PRX+Yv1utLHlMZFgDpQEcR8RSRd0XkiPl4V0Q8K6hvlIjMF5EUEdkvImNP4rRjgP1KqZVmXbKVUvOVUoeqsY8ZwK0lXo/CCDRVopTaCfwGdC7xGdyrlJpn1kcppTYrpW5WShWWs30e8BXQ+fhlWsOkg4JWV/oBXsC3lazjAB4CQs31hwD3HrfOSKAP0FEpNdAs62b+Yp1TckURsYjIlUAQsBV4GuiL8WXdDegNPHN8JUTEAnwP/AU0MesxTkSGVe1UXTYB7UXk/0TkfBHxq+b2AF8CN4iI1bza8QPWVXVjc5sBwGaM99QT+K4a2/sBN5vba2cBHRS0uhICpCql7BWtoJTaqJRaq5SyK6UOAP8DBh232mtKqWNKqfxKjhVltnOnAs8DtyildmF8uU1QSh1VSqUALwK3lLN9LyBMKTVBKVWklNoHfALcULVTdZ3PPmAwRmCZC6Sa7fXVCQ7xwC5gKMZVwowqbrdJRNIxgtunwOcYwbbUZyAiv5tXZvkiMrDE9uPN93APRiAaXWLZdeY2xY+fq3E+2mmusnZZTatJaUCoiLhVFBhE5BzgHSAW8MH497nxuNUOV+FYR5RSTcspjwIOlnh90Cw7XnP+DSzFrBjNMNWilFoLXAcgIr2AORhXLE9WYzfTMb6Uz8X41X9OFbbpoZTaU7JARMp8Bkqpc81l8ZT+kfiWUqrMVZRprlLqP9Wov3YG0VcKWl35AyjEaP6pyGRgJ9BWKRUAPAXIceucSlrfIxhf+MWizbLjHcboCwgq8fBXSl16CsdGKfUn8A3Vb5+fDwwH9lWzP+J4xZ9BtTv2tbOHDgpanVBKZQLPAR+KyEgR8RERdxG5RETeMFfzB7KAHHMI5D1V2HUy0KqK1ZgFPCMiYSISatbny3LWWw9ki8jjIuJttud3Nn/pV8QqIl4lHh4icp6I3Cki4QDmOY0A1laxvgAopXKBC4D/Vme7cvaTgdFk9pGIXCMi/ma/Swzgeyr71hoOHRS0OqOUeht4GKNzNwXjF/n9wAJzlfEYwyezMdrw55TdSxkvAF+YbdvXnWDdl4ENwN8YHc+bzLLj6+kALsMcPYTRN/EpxrDSijwB5Jd4/ARkYASBrSKSA/yI0dH+RgX7qJBSaoNSam91tytnP29gfAaPYQTUZIy+m8eB36u4m+uPu08hpzjwaWc+0ZPsaJqmacX0lYKmaZrmooOCpmma5qKDgqZpmuaig4KmaZrmooOCpmma5nJG39F88cUXqx9//LG+q6FpmnamOf6mUJcz+kohNTW1vqugaZrWoJzRQUHTNE2rWTooaJqmaS61FhREpJmI/CwicSKyXUQeNMuDzen9dpt/G5nlIiKTRGSPiPwtIj1qq26apmla+Wqzo9kOPKKU2iQi/sBGEVmOkQJ4pVJqoog8gZEz5nHgEqCt+eiDkTGzT3UParPZiI+Pp6CgoIZOQ6uMl5cXTZs2xd3dvb6romlaDai1oKCUSgQSzefZIrIDY7KRKzAmHgH4AliFERSuAKYrIxnTWhEJEpFIcz9VFh8fj7+/Py1atECkwg52rQYopUhLSyM+Pp6WLVvWd3VqTJHdToH97M4JZrM7Sckp5HDaMeKOrKKNV3ssYkEpJ468TAry0lif+ytZjkzszkLsziLAiSgnFhSCA6tyYFEOTi3b+amxIFiVYDFr4RSFA1VjNRLAimBRggBOAQcKZzWPYEHoYPOluy0ADywoFPut+WzwyCJHyp+XqqVfOx6/6dNTP4nj1MmQVBFpAXTHmEawcYkv+iSMicPBCBglJ1CJN8tKBQURuQu4CyA6OrrMsQoKCnRAqCMiQkhICCkpKfVdlRqz/1gaI7+9nvz0zhSlXljf1alx7o1W49Pod7yUA19lJ9ImXJQSjLfTWmo9bykk3/sIn4fbiXd346KcXF5LScMDyBdhXHgov/t44+104qkUHiijLbrEfzslgkLqMSQY87s6RXBgtJVbFWbQqhkKcCI4BJwYMzFZlap2u3yBCL95puPjVPTJc7DHw8JhDwvuTkWQs/x30CN33ynWvny1HhTMqQfnA+OUUlklv6yVUkpEqvVvRik1BZgCEBsbW+62OiDUnYb0XtsdTm5d8CxOt2S8w9IY0+N6Atwbl7tucuFu7MpGE6+OVdr3vtz1rE2fxTVRr+Jh8a7JapelFO6OHNwcBYgyfrU3yt3HobT5vO+2l04FhYQ4HBRaLGzw9+SYTzZvHrPRwvxBetQCUwPcWegjRIk/V/u1ZT5/kxrWnqfCLubltGX8lXuICR3v4MoWl4C7t/GwuoPF3fzrBg3o30ZtcyonG5I2sHDvQlbFr6JNUBv+23oEFza/EH8P/zqtS60GBRFxxwgIM5VS35jFycXNQiISCRw1yxOAZiU2b2qWnVHS0tIYMmQIAElJSVitVsLCwgBYv349Hh4e1drfzp07ue2229i0aROvvPIK48ePr/E6a4ZHF31LuvU3ugZdwK7sNaR5LOLh814ps97G5I3cvfxplHLy5YC36RDUuuKd5qZQdGANI5KnkYKNzttvYqC9lsZ3KAW2PMhLA2fpJoctnh48FdmYnuLHlJj78GgSC+Ed2JKxmwd/fpC7vG3c1fUu1iat5Y8jfwAwuuMo7o25F283b3rsXchza57jxvhpOJWTiQMmcmmrU5qITivBIhZ6R/amd2Tv+q5K7QUFMX5CTgV2KKXeKbFoIXArMNH8+12J8vtFZDZGB3NmdfsTTgchISFs2bIFgBdeeAE/P79T+iIPDg5m0qRJLFiwoGYqeBbKLrBRZHdWus6vexJZlvwBfl5hfNLjRj7e4WDa3u+51acV53gEQWEOFOWwI2kT92euJ9JuI0+Eh1fcw5wjSQRUcIkPMCvAn4SQRgiwLiicgZ7NK1z3lLn7cMTDkzdydnLEkUsn70jaezfmo6N/EOnViHeHf4WHV5Br9ZjwGGYNn8XYn8by9sa3ifCN4M4ud3JFmyto5v/vb7QRrUcQ5BnEm3++ybie4xgSPaT2zkGrV7V5pdAfuAVj1qktZtlTGMFgrojcgTFxevFsWYuBS4E9QB5wWy3WrU6tXLmS8ePHY7fb6dWrF5MnT8bT05MWLVpw3XXXsWTJEry9vfnqq69o06ZNqW3Dw8MJDw/nhx9+qKfan9l+2b+d+1b9h7wD9+IsjKpwPY+wH/EMTeUdmuAzdRh3WCzMaxrFpHWv8UGy0Wey392NMZER+Fk9mBJ9Gcnegdy2bxbPdrmAd5tfWW5TWqbFypTtk+gf1g2b08b6oiy4vOY7B8Fogpi9czbvbnoXgK6hXVmaFse89L8J8Ajgw6EfEVQiIBSL8oviy0u/5EDWAdoGtcVqsZZZB2Bg04EMbDqwVuqunT5qc/TRairOr1HmZ4Y56ui+mqzDi99vJ+5IVk3uko5RATx/eacqr19QUMDo0aNZuXIl55xzDqNGjWLy5MmMGzcOgMDAQLZu3cr06dMZN24cixYtqtH6nu1+2rcBsdi5tNNBhgSfU2qZRTnwzzuER/YOHrCv4tLsHPrn7IChLxLYajC37/+e9/bM5ZVBdxKXfYCt6f8Q5BnElEu+IDKwJZHAQ8FRvLnhTaZ7Orm1061ljv/phrfJtuXyUM+HWHV4FR9s+YD0gnQaeTWq8jnk2/N56OeHGNVpFOdGnVtm+YHMAyw/uJzF+xezJ2MP/aP681y/54jyi8KpnBzIPECwV3C5AaGYl5sX7YPbV7lONUnZ7aAUchLDmp2FheT/9Rc+vXo1qP6t+nRGJ8Q7EzgcDlq2bMk55xhfSLfeeisffvihKyjceOONrr8PPfRQfVWzwUpM2QyAb8pXXBn3frnrrPDxxt44jKtaDocLXgbvIABuDm/H7CO/MOfQMrqEdeGebvdwWevLSjWr3NLxFjYd3cS7m97lvCbn0bpE/0JCTgIzd8xkROsRtAtuR749nw+2fMD6pPUMazGsyucwc8dM1hxZg0KVCgpKKR5a9RArD60EoFtYN14b8BrDWw53fUFaxEKroFZVPlZdydu4kaylSynYuo2CuDgQwbt7DL69e+M/dCiebduecB/K6eTI+EfJXr6cqLffInD48Bqpm1KKwp07sQYH4964/IEGDVmDDgrV+UVfX0r+utG/dGpeRu4ecIO4Rk1gwMTjlgo0asHvB7/D99AKul7yf8boGZO3mzfzLp8HUOGvbBHh2b7PsvG7jTz3+3NMv3g6VouVQkchT/72JFaxcn/3+wHoFNoJHzcf1idWPShkFmby2dbP8LB4sDZxLSl5KYT5GAMXtqZuZeWhldzY/kZu73w7Eb4R1Xtz6kDxHPDF/7aV00nq5MmkfvAh4umJV8eONLrhBpTTSd6ff5Ly3iRSP5pM1JtvEnBx5e/R0bffJnv5ciwBAaS8N4mACy9EqjmQoyR7aioZ335L5rcLKNq3D/H2JvzR8TS68UZEhML9+0l9/wPE3Z3Iia812P+vDToonA6sVisHDhxgz549tGnThhkzZjBo0CDX8jlz5vDEE08wZ84c+vXrV481bZgyVBoAB22Z5HYYjq+7b6nlSil+X/8svSN6424p23xRWZNLsRDvEB7v/ThP/vYks3bO4uYON/PcmufYfHQzbw16y/Vl7W5xJzYilnVJ66pc/6nbppJjy+GtQW/xyC+PsGT/EkZ1GgXA3F1z8XHz4cEeD5Y5r7pgS0rCnpKCMycHZ0EBPj16YA0MdC3P/uknkp5/AdzdCLhoGH6DB5P2+Wfk/vIrgVeMIOKFF7B4lx6ea09JIX7sgyQ89BD2Y88QfNNN5R47fc5cjk39jEY33YTfoIEcvnsM6V9/TfDNN5/UuSi7nQPX34AtIQHvnj2JeOF5slesJHnCS+SsWIl7dDMyvp5nDLO12/EdMIDAy2rmyuR0o4NCLfPy8uLzzz/n2muvdXU0jxkzxrU8PT2drl274unpyaxZs8psn5SURGxsLFlZWVgsFt59913i4uIICAioy9M4IzmdTtItuUTaFYlusCNtB7ERsaXWOZR9iIScBG7rdGrjGoa3HM7ifYuZtHkSB7IOsHj/YsZ2H1vmiqB3RG9+jf+VpNykE/6yT8pN4qsdX3FZq8u4qMVFdNrWiUX7FjGq0ygyCzNZemApl7e+vF4CQsGuXey/8ipw/juqS3x8CLrmaoKuuYZj06eTOW8+nh064B4ezrGZMzk2bRq4uxPx/HME3XBDub+03cLCiP5sKgkPPUzyhJco3LEDr06dcQsLBauVwn92U7hzB1lLl+E7aCCNn3oSrFZ8evcm9aPJBF4xEqtf9d+PnN9+w5aQUKoZKuj668mYM4fkN95ErV9Po+uvJ+Tuu4m/916Ovv46foMHYfXzq3CftuRkUj/4gNy164j+/DM8mjatdr3qhVLqjH307NlTHS8uLq5M2emqefPmKiUlpb6rccpO1/d8T2qi6jyts3rp496q87TO6ottX5RZZ2bcTNV5Wmd1KPPQKR8vMSdR9ZnZR3We1lk99dtTyul0lllnZ9pO1XlaZ7Vg94IyywrsBeqdDe+oT//+VP186Gf1+K+Pq5jpMSo+O14ppdSM7TNU52md1d70verLuC9V52mdVVxq/bz3CY89pnZ076GyVqxQOevWqZy161TCY4+puE6dVVy79iquQ0eV/PY7yllYqJRSyp6ZqTIXL1b5O3dVaf9Om00dee55Fde+g7G/Eo/dFwxRCY89puzZOa718/76S8W1a6+OTnpfKaWUo7BQFcXHV/l8Dt13n9p1bn/lLCoqs6woMbHUvvL++kvFte+gkl59tdx92bNzVPJbb6kdXbupHZ27qB1duqrDYx+scl3qSIXfq/pKQWuw/kz4B4BOXk0J985gx7EdZdb548gfNPNvRrOAZmWWVVeEbwQv93+Z1QmrebrP0+X+Em7bqC2NPBuxLnEdV7S5otSyX+N/5bNtn5Uqu7nDzTTxawLAxS0v5q0Nb7Fo3yJWHlpJl9AudAjpcMr1ri5bUhKZPyym0U034j/k34GEvn16E/bQQ2T98APe3Xvg06O7a5k1IICASy6p8jHEzY3IF18g4umnsB87hj0lFWUrwrNNG6zlXCV7d+2K/7BhpE2dStaiRRQdPgxOJ1FvvXXCZh57aio5q34heNSockdAuUeUvqLz7tqVoGuv5diXMwm86iq82rX7d19paRy+8y4KduwgcMTlhD4wlsyF35E66X1y163Ht0/935x2Ijoo1KMDBw7UdxUatB1JRhBoGdKOjt7ZxKXFlVpuc9hYn7Sey1tfXmPHHNp8KEObD61wuUUs9IroxbqkdSilSgWO3+J/w9/dn8VXLeZA1gESchI4v9n5ruWh3qH0jerLlzu+JN+ez4RzJ9RYvavj2IwZ4HQSPKrsEFz3iAhC7rijxo4lHh64R0SU+WIuT/gjD2NPTcUtNJSA4ZeS9eNSUidPJuDSSxBLxXeRZ363EOx2gq6+qsr1CntoHNnLlpHwyCOEP/wwfoMGYUtM4vAdd2BLTqbZx5PxM/sOQ26/nYx580h+9VVafjMfsZa+D8SRk4MjLQ336OjTovNaT7KjNVhH0rZhVYp20d3pGNKR/Zn7ybPluZZvSdlCnj2v3LH/talPZB+O5h1lT8YeV5lSitUJq+kX1Y8gryBiwmMY3mo4Pu4+pba9rNVl5Nvz8Xf35+KWF9dpvcH4AsuYM5eAi4fh0bRJnR+/Mh7R0bSY+SVN33uXsLFjCb33Xor27iV75UrXOrajR9k34gpSJk1yNZdkfPMN3jExeLauJF3JcdwaNSJy4ms4s3OIv/c+9l54EQdvvBF7ejrRn011BQQAi5cXjR97jMJdu8j4eh729HTSZ83i0H/vZPegwfwT24u9wy4macIE12it+qSDgtZgpeYfJMpuxzu0DR1COqBQ7Dy207X89yO/4yZu9I6o20v6C6IvwCpWvt/3vatsV/ouUvJTGNB0QOXbNruAAI8Armx7Jd5utZxYrxwZ8+bhzMkh+LbTP+FAwMXDcI+OJu3j/xkBwOkk8YknKNy9m9SPJpP4zDPkb9xI0d69BFbjKqGY/+DBtFm5gibvvYd7kyaIpyfNZ8zAp0fZ+cH8hw3DO7YnyRMnsnvAQJJenIDtyBF8+/Yl7OGHCbr2WjJmzSbtf/9zbZO7fj37Lr+ctGnTTuVtqDbdfKQ1WFnOVNra7NCoJR1VEQBxaXH0aGz8p12TsIauYV3x86h4BEltCPUOZUCTASzau4ix3cfiZnHjt/jfADivyXmVbuvj7sPCkQsJ8Kj70WeOzEyOTZ+OT2ws3l261Pnxq0vc3Ai5878kPfscuWt+p3DnDnJ//4OICS9iT0om9aOPyP5xKeLjQ8AlJ5fcT9zcCBh2EQHDLqp8PREinn2OxOeexbdXLwIuuwzPdu1K3b/hLCwg5d33sAY1wpYQT9qnUxEPD46+8SZe7drhW0dD1vWVgtYgOZ1OMqy5RDks4B1EuE84od6hrs7mjckb2XFsR503HRW7os0VpOSnuDKSrk5YTYfgDoR6h55w2xDvENytNTvTXeZ333HwllEU7tlTqlw5HGQtXcbh++/nn/MGYD+SSMjdd9XosWtT4BVX4Na4McmvvsrRd9/D/+KLCbr2WsLGPkDEC8/jzMsj4NJLTmoYa3V5tTuHlnPmED5+PF7t25e+cdViIerll/Ht35+kF14g7ZNPCbr2WtqsXIFHy5YkPDIeW3JyrdcRaneO5s9E5KiIbCtRNkdEtpiPA8WJ8kSkhYjkl1j2cW3Vq7alpaURExNDTEwMERERNGnSxPW6qKjopPY5ePBg2rVr59rP0aNHy6wzbdo0RIQVK1a4yhYsWICIMG/evJM+nzPV7rQkiixOGlv+/UXdMaQjcWlx7M/cz9ifxtIioAU3tL+hXuo3qOkggjyDWLh3IZmFmWxJ2XLCpqOa4CwowH7cpEiZ33/PkSeeJG/DBg5cdz3Z5r+h/L//5sC115Hw4IMU/PU3wTfdRMtv5uM3oPbrWVMsHh6E3HE7Rfv24R4eTuSEF11fxo1uuIFWPywi4umn67mWBvHwoMl77xF0/fU0/fADIie8iFtoKE3fn4QqKCDhwXGok/wOqY7abD6aBnwATC8uUEpdX/xcRN4GMkusv1cpFVOL9akTNZ06u9jMmTOJjY2tdJ0uXbowe/Zshg41Rr/MmjWLbt26nfKxz0QbjhjDUaO8Il1lHYI7sDphNfesuAc3ixuTh04m0DOwol3UKnerO5e2vJR5/8yjV0QvnMrJgCa1/2Wb9OIEMr/7joDLhhM6ZgyF//zDkcefwKd3byInvEjC+EeJv/8BfPv3J/f333ELDSXq7bcIuPjiMqNmzhRB115L4e49BF1/fZnhrJ6tTq+8UFY/XyJffKFUmWerVkS+8jIJDz1MyuTJhD/4YK3WodauFJRSvwLHyltmzrVwHVD2Ft4GaOXKlXTv3p0uXbpw++23U1hYCECLFi147LHH6NKlC71792bPcZfu1TVgwADWr1+PzWYjJyeHPXv2EBMT41q+ceNGBg0aRM+ePRk2bBiJicZ0FZ988gm9evWiW7duXH311eTlGSN0Ro8ezdixYzn33HNp1arVGXXFEZdsBIWWIf8mVusY0hGncpKWn8aHQz6kqX/93mF6RZsrKHIW8e6mdwn0DKRLaO220zsyMsj64Qc827Qhe/kK9g2/jIRHxuMdE0Ozjz7Eo3lzmn85g8CRI8n94w8a3fIfWi1ZTODw4WdsQACweHsT+dIEvDuf/rnQKhJwySX4X3Ix6V9Mx56eXqvHqq+O5gFAslJqd4myliKyGcgCnlFK/XbKR1nyBCRtPeXdlBLRBS45PrFaxWoqdfZtt92G1Wrl6quv5plnnil3PLOIMHToUJYuXUpmZiYjRoxg//79ANhsNh544AG+++47wsLCmDNnDk8//TSfffYZV111FXfeeScAzzzzDFOnTuWBBx4AIDExkdWrV7Nz505GjBjBNddcU513q94kHNuBm1K0bfLvlVKP8B50CO7AfTH30Tm0cz3WztAhuANtgtqwJ2MPl7S4pMJ5DGpK5sKFqKIiot54HbfwcI5N+4Kiw4eIfOklLL5Gm7rF05Ooia8R8ewzrjLt9BA65h6yl/zIsenTa/Vqob46mm+k9FVCIhCtlOoOPAx8JSLlDq8QkbtEZIOIbDgTJowvL3X2r7/+6lpeMnX2H3/8Ue4+Zs6cydatW/ntt9/47bffmDFjRoXHu+GGG5g9ezazZ8927Rtg165dbNu2jQsvvJCYmBhefvll4uPjAdi2bRsDBgygS5cuzJw5k+3bt7u2GzlyJBaLhY4dO5JcRx1dNeFY3n6a2O14hv47aVGQVxBzL5/LoGaDKtmy7ogII9uMBKj1/gSlFOlz5+LVtSte7dvjFhxM+MMP0fT//q/c/D06IJx+vNqdg/+FF5I+40scWTU7T0xJdX6lICJuwFVAz+IypVQhUGg+3ygie4FzgA3Hb6+UmgJMAYiNja38To9q/KKvL8enznY4HPTsabw1I0aMYMKECTRpYtwk5O/vz0033cT69esZNWpUufvr3bs3W7duxcfHxxWIwPhS6NSpU7mBZ/To0SxYsIBu3boxbdo0Vq1a5Vrm6elZah9niixnCh1sdghuWd9VqdQ151xDvj2/0ruga0L+5i0U7dlL5Msv1epxtNoVes8Yspcv59iXXxJ27721coz6uFIYCuxUSsUXF4hImIhYzeetgLbAvnqoW40rmTobKDd1dvHffv36YbVa2bJlC1u2bGHChAnY7XZSU1MBowlo0aJFdO5cedPHxIkTefXVV0uVtWvXjpSUFFdQsNlsriuC7OxsIiMjsdlszJw5s2ZOvB45nU6yrDk0szvB7/SbY6AkX3dfxnQbU+s3omV8/TUWH59q5R/STj9eHTviN3gw6V9Mx5GTWyvHqLUrBRGZBQwGQkUkHnheKTUVuIGyHcwDgQkiYgOcwBilVLmd1GeaU02dXVhYyLBhw7DZbDgcDoYOHepq/6/IJeX8x/fw8GDevHmMHTuWzMxM7HY748aNo1OnTrz00kv06dOHsLAw+vTpQ3Z29qmfeD3alXoEm8VJY/GHSnLenCmUUqR9/DH+wy7Gs1X1r3wcWVlkLVlC4IgRulmoAQi99x4OXHc96bO+IvQE3wUnpbIUqqf7Q6fOPj2cbu/5Sz8b6bAXT7mwvqtSI/I2b1Zx7dqrxJdfqXS9lP9NUamffqqcdnup8rRp01Rcu/Yqb+u22qymVocO3T1GJb/55qnsQqfO1hq+PFsh//3uFf7OWUC03U6fxqfPEERbUhJZPywmeNQt1Z6gPmvJEgDy//qrwnVyVq8h5Z13AMhds4aoN97A4utLyrvvcmz6DLx79Dijh2RqpTX96MNKM7+eCh0U6pFOnV1z9h1L5roFd1BoPUhLax/mHJyHT5f29V0tl+TXJpK9dCmO7CzCzeHIVaGcTrJ+XApAwY4dOAsLsZTo/Adw5uWR9MILeLRoQfDoW0l+bSL7r7wK8fHGdvAQjW66ifBHHq7J09HqWW0FBNC5j7QG4q01X1FoPciNLZ7i+15X4aMUNDo9Rh4V7t1L9rJlWENDSfvfFHLXVn2O5vzNm7EnJxNw6SVgs1FQYrhwsZQPPsQWH0/EhBdpdMMNtJg7x+g7cDiJnjaNiOee1X0JWpXpoKA1CHszdyMOP57K3Qxzbga/xtCk54k3rANpUz5BvLxoOWc2Hi1acOSxx6p8V2rW4iWIpydh5tVF/uYtpZbnb9/OsWnTCLr2Gnx7GynAvdq1o9Wi72n94xJ8+/apyVPRzgI6KGgNQmbhbmKLMuDPTyD2DrhvHfiGnPJ+bUlJZRLIVUdRfDyZixbR6LprcW/ShCbvvI0jPZ3Ep54+4X0fyuEga9lS/AYNwiM6Gvdmzcg382qBMUgk6cUJWEOCCT8uv5a4uSFuunVYqz79r0Y74xXYiiiwJNOhKA/u/Amiup94oypQSnHw5v9gM5tvQkaPRjw9yfxuIVmLFmEJCqTJW29VmlQt7dNPwWIh+PbbAfDq0IGwceM4+uabFGzbjneXiu85yftzA46UVKPpCPCOiSFv7VrXNJ4FcXEU/P03jZ99Bmtg/ST20xoefaVQC5KTk7npppto1aoVPXv2pF+/fnz77bc1fpydO3fSr18/PD09eeutt2p8/2eKtYf/wWFRNJOgcgPCySYQKzpwAFtCAt7dupGzYiX7r7qafcMvI23qVDxat8aelMyBa651dQQfz5Z8lMz53xA0ciTujRu7ygOGGxO65G/eXOnxs5YsQXx8XFM7enfrhj0lBfuRIwBkzv8G8fAg8LLLTur8NK08OijUMKUUI0eOZODAgezbt4+NGzcye/ZsV56hmhQcHMykSZNqJDX3meyPw0bna6ugdmWWFe7eze5z+5O3aVOF2yunk8N3jyH7559LleetMzqEo155mTa/rKLxU0/R+KknabvqZ6I/mULLb+bj2bYtCePGkfLBh2X2mz7rK5TDQcid/y1V7h4RgVtERKmmoDJ1stnIXrYM/8GDsXgbdzt7mxlv87ZswVlYSOYPP+A/dKi+StBqlA4KNeynn37Cw8Oj1F3LzZs3d2UddTgcPProo/Tq1YuuXbvyP3NO1lWrVjF48GCuueYa2rdvz80333zCNufw8HB69eqFezXHvTc0/yRvxqoUHZv3LbOscP9+UIqcEkkIj1d08CA5v/xC+nGJBnPXrsOtcWPcmzfH6u9P8KhbCB41CrewMMD4cm8+Yzr+l1xM6scfY09Lc22rlCLr+0X49uuHR3R0mWN6d48hb0vFVwqZC7/HkZ5O4BUjXGVe7c5BvLzI3/IXOT/9hDMz86TmFta0yjToPoXX179eaqL2mtA+uD2P9368wuXbt2+nRzkTdxebOnUqgYGB/PnnnxQWFtK/f38uusiY33Xz5s1s376dqKgo+vfvz5o1azjvvMrn7NXgaM52oh12/Jr1KrPMbuaNyttQJreiS0FcHAC569bjyMjAGhSEcjrJW78evwHnlZumvJh4eBB2//1kL/mRzIXfE3LbaMBoGrIlJBD6wP3lbucTE0P2kh+xJR/FvXF4qWWqqIjUjz7Cq3NnfAcO/PdY7u54d+5M/pYtFB04gFtkJL59ywZCTTsV+kqhlt13331069aNXr2ML6xly5Yxffp0YmJi6NOnD2lpaezebUwr0bt3b5o2bYrFYiEmJkbf3FZFGSqJc4qKjLkujuNINX69F/z1N05zcqPjFe7YYa7sINvMEFu4ew+OY8fw6XPiL13P1q3x6taVzG++cV3dZS1ahHh64j/0wnK3KW4KKq8JKeObb7ElJBA29oEyAcm7ewwFO3aQu2YNgSOvOKMnv9FOT7WZEO8z4DLgqFKqs1n2AnAnUDzG7yml1GJz2ZPAHYADGKuUKr/3rhoq+0VfWzp16sT8+fNdrz/88ENSU1NdU2kqpXj//fcZNmxYqe1WrVpVKk211WrFbrfXTaXPYMk5mWS5FdBMeYN3UJnlxVcKymaj4O+/8elV9mqiIC4Oz44dcBxLJ3v5CoJGjnT1J/j26V2legRdeRVJL7xAwbbteLVvR9aSH/G74PwKJ4T36tAB8fAgf8sWAoZd5Cp3FhaSOnky3jEx+JYzF7J3TAyY/y6CrryySnXTtOqozSuFacDF5ZT/n1IqxnwUB4SOGNlTO5nbfFScSvtMc8EFF1BQUMDkyZNdZcXTWwIMGzaMyZMnY7PZAPjnn3/Iza2dFLhng1/2GzPrNfNuXu5ye1oablGRIELun3+WWa6UoiBuB14dO+J/4YXkrl6NMzeX3HXrcG/WDHdzLosTCbj0EmO46rffkLNmjdEfcPnlFa4vHh54depU5kohY85c7MnJhD04ttxmq+IrDJ/Y2HL7KjTtVNXalYJS6lcRaVHF1a8AZitjsp39IrIH6A2UPxXZaUxEWLBgAQ899BBvvPEGYWFh+Pr68vrrrwPw3//+lwMHDtCjRw+UUoSFhbFgwYJK9/ncc88RGxvLiBEjSpUnJSURGxtLVlYWFouFd999l7i4OAICyp20rkHadNjoK2gfWf7dy/bUFDxbtsIaEEh+Of0K9sREHBkZeHXsiGebNqTPmEHOL7+Q9+ef+F9UftNPeawBAfgPHUrmD4uxp6RiDQzE7wT9Qd7du5M+YwbOoiIsHh448/NJnTIFn1698Kmgr8AtJITQB+7Ht9+5Va6bplVHfXQ03y8iozBmVXtEKZUONAHWllgn3iw7I0VGRjJ79uxyl1ksFl599dUyk+AMHjyYwYMHu15/8MEHrucTJkwod18RERG1MtT1TJKQugkfp5Nz2gwsd7kjJRXPFi3waNmSjPnzUTZbqSylBWZ/gleHDnh37Yo1OJiUjz7CmZWFbxX6E0oKvOpKsn74gezlywm6/nrEw6PS9b1junHss88ojIvDOyaG1I8m40hNJWzSe5V2bofdd1+16qVp1VHXHc2TgdZADMa8zG9Xdwdn2hzNWu1KLTpA2yIbbk3KjvhSSmFPS8MaGopPbCwqP79MQrmC7XFgseDVrh1iteI/5AKK9uwFwKeK/QnFfPv2xS0yEoDAy4afcP2S9x0U/PMPaZ9/TuCVV+JTyeg1TattdRoUlFLJSimHUsoJfILRRASQADQrsWpTs6y8fUxRSsUqpWLDzPHi2tnJ6XSSZskk2u5Wbp4jZ04OqrAQt5BQfHoZHf3HD00t2LEDj5Ytsfj4AOB/odFk5NGqFe7hpYeKnohYrYTcdpsxd0HPEyfjcw8Pxz0qivzNW0h64UWsfn6EP/ZotY6paTWtykFBRHxO9WAiElni5ZXANvP5QuAGEfEUkZYYczSvP9XjaQ3btqOHyLc6aereuNzlxSOP3MJCcQsJwaNVK/L+PC4oxMXh1bGj67VP375YGzXCr5yRP1URPOoWWnw1s8r57r1jYshevpz8TZsIf/RR3Bo1OqnjalpNOWGfgoicC3wK+AHRItINuFspde8JtiszRzMwWERiAAUcAO4GUEptF5G5QBxgB+5TSjlO8py0s8SavcbvhhbBHctd7igOCiHGVYRPbCxZixejHA7EasWeloY9OblUULB4eNDq+4VY/PxqufYG7+7dyVq8GO/YngRepYeYavWvKh3N/wcMw/g1j1LqLxEpv1evBKXUjeUUT61k/VeAV6pQH+0sl5SdzvM/T+XPlK9xtyq6tx5c7nrFaSesoaEA+PTqRcbcuRTE7cC7S2cK4v7tZC7JzVy/LvidP5jMRd8T+eKLlXYua1pdqdI1rlLq8HFF+le8Vi8mr13ChV9fxO/pX9DeXsCUpKNEtim/qceeYl4pmF/yvv36Ij4+JD3/PM7cXFd6C6+OHcrdvi54NG1Kyzlz8Gzdut7qoGklVSUoHDabkJSIuIvIeGBHLdfrjFZXqbOnTZuGiLBixQpX2YIFCxAR5s2bV+PHq282h5OZWz4ngEJmJyTxVXY+sRe9Bf4R5a5vT0sFqxVrUBBgBIem//cOBTt3kjD+UQq2bcO9aVOsZ9F9HZp2IlUJCmOA+zDuG0jAGE6qB0pXoC5TZwN06dKl1D0Rs2bNolu3brVyrPr27aYE3K3/0LOwkE5DXoYH/4Ieoypc356ailtISKlOX79Bg2j8zNPk/Pwz2cuXl+pP0DTtBEHBTDXxnlLqZqVUY6VUuFLqP0qptMq2O5vVZepsgAEDBrB+/XpsNhs5OTns2bOHGHP8O8DGjRsZNGgQPXv2ZNiwYSQmJgLwySef0KtXL7p168bVV1/tSsUxevRoxo4dy7nnnkurVq1OmysOm8PJlJ/Xk+rhoGNoR+hzN7h7V7qNIzUNa2jZoarBN91E8G23AeigoGnHqbSjWSnlEJHmIuKhlCqqq0rVlKRXX6VwR82mzvbs0J6Ip56qcHldp84WEYYOHcrSpUvJzMxkxIgR7N+/HwCbzcYDDzzAd999R1hYGHPmzOHpp5/ms88+46qrruLOO+8E4JlnnmHq1KmuwJWYmMjq1avZuXMnI0aM4JprrqnWe1Qbvt2cQJPCpSQDHVuXl1KrLHtqaoWdxuGPjsezdSv8zj+/BmupaWe+qow+2gesEZGFgCtzm1LqnVqrVQNy3333sXr1ajw8PPjzzz9ZtmwZf//9t+sXeGZmJrt378bDw8OVOhtwpc6uynwKN9xwA5MmTSIzM5O3337blUJj165dbNu2jQvNG7IcDgeR5h2327Zt45lnniEjI4OcnJxSWVtHjhyJxWKhY8eOJCcn1+j7cTJsDicf/LSHYQHb2AR0bFe1oZv21FQ827Ytd5lYLASdBsFO0043VQkKe82HBfCv3erUrMp+0deW+kid3bt3b7Zu3YqPjw/nnHOOq1wpRadOnfjjj7J5BUePHs2CBQvo1q0b06ZNY5U5jwBQqh5VacKqDVM2LOaX3QeJcjuXtNwiMo4dJbdZMmGWRoT6nvhO4+IUF3U5vFTTGoITdjQrpV5USr2Ikafo7RKvtXLUV+rsiRMnlkmy165dO1JSUlxBwWazsd3M/ZOdnU1kZCQ2m42ZM2ee8vFr0uGMNN7/+wX+LpjK6j2J7EzM4pFmu9nh4UaH4PZV2oczMxNsNtzK6VPQNK1iVbmjuTMwAwg2X6cCo5RS2yvd8CxVl6mzS7rkkkvKlHl4eDBv3jzGjh1LZmYmdrudcePG0alTJ1566SX69OlDWFgYffr0ITs7+5TOuyY9tOwtsOYD8Np/vBjcbDD50yfxjtODIVFVy1x6/I1rmqZVjZyoeUBEfgeeVkr9bL4eDLyqlKr3hO6xsbFqw3EJznbs2EGHDvV3M9LZqCbf8z/j93Db8muIcu9DjnUrFzS7gJe7j+Ov9zvxn8hw3j3/XYZEDyl32+L0FQC5a9dxaPRooqdNw7dvnxqpm6Y1IBXePl+V+xR8iwMCgFJqFVD+HIOadoqe+Pl1wMJ73UYw2BrEz/sWY1s4lh3uxkVtxwryHOX8+iv/9O6DLfkoYN64hpEMT9O0qqtKUNgnIs+KSAvz8QzGiCRNq1Hztq7hqHMtPX2H0WHJAww9+BdZysaf8b+xIzSaIM8gInzLv3s5f8tfOHNzyfnZ+P1yfDI8TdOqpipB4XYgDPgGmA+EmmWVEpHPROSoiGwrUfamiOwUkb9F5FsRCTLLW4hIvohsMR8fn9TZaGe0dzdOBocf73lkQX465960EG83b1aedxc7gpvRIbhDhUnjiuKN9Fw55igqe2oauLtjCQysq+prWoNQldFH6UqpsUqpHkqpnkqpceYUmicyDTj+LqPlQGelVFfgH+DJEsv2KqVizMcYTkF9DaM8G9XUe51TaCfDfoAOHq0I2jITet+NV5OeDGgygBWHVrA7YzcdQyq++9h22EgjkvvHHzjz8/9NcaEzj2patZwwKIjI8uJf9ObrRiKy9ETbKaV+BY4dV7ZMKVU8+H4txgxrNcrLy4u0tDQdGOqAUoq0tDS8vLxOeV8/74pH3DPpn7sL/BrD+cY9JkObD+VYwTHsTjsdQiruzC6KP4x7VBSqsJDcdesqvZtZ07SKVeXmtVClVEbxC6VUuohUb57C8t0OzCnxuqWIbAaygGeUUr+Vt5GI3AXcBRAdHV1medOmTYmPj0fP31w3vLy8XHdhn4rFO40Rzm2zEuDi98HLyFw6sOlAPCweFDmLKuxkdubn40hJpdG995I2bRo5q1ZhT0vFPawm/plq2tmlKkHBKSLRSqlDACLSHGPmtJMmIk9jzLBWfNdUIhCtlEoTkZ7AAhHppJTKOn5bpdQUYAoYQ1KPX+7u7k7Lli1PpXpaHXM6FWsP7YIwaB7SATpf7Vrm6+7LuU3OZVPyJpr6lx98ig4b/QkerVvh1/9cclb9gnLYdbI7TTsJVQkKTwOrReQXjLGtAzB/qZ8MERkNXAYMUWYbj1KqECg0n28Ukb3AOcCGivajNRzbjmTip+LIAJp1uRGO6wd4us/TpOSlVNg/YDPTkns0a4bf4PPJXm7ML+EWopuPNK26ThgUlFI/ikgPoC/GFcI4pVTqyRxMRC4GHgMGKaXySpSHAcfMrKytgLboYa9njZ92HiXScw/icBDQ6eoyyyN8IyocigpgM68U3Js1wz0qylWu+xQ0rfoq7Gg2U2YHAphBIBe4CBglIh4n2rGIzAL+ANqJSLyI3AF8gJFUb/lxQ08HAn+LyBZgHjBGKXWsvP1qDc9PO5Jx90wh2uoDPsHV3r7ocDwWX1+sQUG4hYbi1bUrgM57pGknobIrhbnAlUCmiMQAXwOvAd2Aj4D/VrZjpdSN5RRPrWDd+Rj3QGhnmaPZBdiP/E1SGyd9g05unmLb4cO4N2vmal7yGzyIgr//1lcKmnYSKgsK3kqpI+bz/wCfKaXeFhELsKXWa6adFVbtSuFCt7V85uZGdJOqJbs7XlF8PJ6tWrleB119NbaDh/Dq1KmmqqlpZ43KgkLJXr0LMG80U0o59Q1B2ql4ctnnpKUHEmRtzcYDx3jWawPgZow8qibldGKLj8dv0CBXmXvjxkS9PrEGa6xpZ4/KgsJPIjIXY7hoI+AnABGJBM64qTm104NSiu8TPoSC5vhn3Etr50Hs7ulAGNEBZe87ORF7SgqqsBCPZjV+H6SmnZUqu6N5HEa+owPAeUopm1kegTFMVdOq7VB6OmIpxM1nHyvviOTLnns46O4OQLR/9YOCa+RR02Y1Wk9NO1tVeKVg3kMwu5zyzbVaI61B25p8CAAHdjZ8fj4D8ws41KIjwV4e+Hn4VXt/RYeL71HQVwqaVhOqcvOaptWYPcl7Xc9Xd76UgU2HcfDwdzS3up/U/myHD4NIqfsTNE07eVVJna1pNeZoShwAzT1DWFN4FLrdwKG85JNqOgIjEZ5bZATiccJbZzRNq4JqBQUzQ2rX2qqM1vBlZu8H4MpmQzmUfYhdx3aRkp9C84DmJ7U/2+F4PHR/gqbVmKqkzl4lIgEiEgxsAj4RkXdqv2paQ5RblIi/w8mF7Yx0FrN2zgI4qZFHYCTDc4/WQUHTakpVrhQCzWylVwHTlVJ9gKG1Wy2tocpX6YQ4FNGhHYj2j+aHfT8AVOtKoXiuDGdeHo7UVH2loGk1qCpBwc28N+E6YFEt10drwGwOJwWWfILxBKB/k/4UOAoAaOZftS/2tKlT2T1wIJnfL3KNPHLXI480rcZUJShMAJYCe5RSf5pZTHdXZecVzNMcbM7mttv828gsFxGZJCJ7zDmce5zMCWmnryMZ+WS72QlyMybQOa/JeQCEeofi6+7LsS++IPH5F7CnpZW7vSM7m9SP/4czK5sjjz5KwoMPAkbKbE3TakZV5mj+WinVVSl1r/l6n1KqbH7j8k2j7DzNTwArlVJtgZXma4BLMFJmt8WYr2FyFY+hnSEOJx/lmFUI9TZmRIttHIu7xd018iht2hdkzJnD3kuHk/711yins9T26TO/wpmdTfMvZ9D4ySewJScbw1F1UNC0GlOVjuY3zI5mdxFZKSIpIvKfquy8vHmagSuAL8znXwAjS5RPV4a1QJDZbKU1EAmHNuEQISrI6D/wcffhrq53cfU5V2M/dgx7YiJBN1yPZ9s2JD37HIfvuhtngdG85MzL49i0afgOGoh3ly4E33orrb7/nmZTpuDWqFF9npamNShVaT66yOxovgwj5UUb4NFTOGZjpVSi+TwJaGw+bwIcLrFevFmmNRCpqVsBiG7c3lU2ptsYRrQeQcF2Y47mgEsupfmMGTR+7lly16wh4cFxqKIi0ufOxZGRQeiYMa5tPZo2wW/AeXV7EprWwFXljubidYYDXyulMmsqS6pSSolIteZ7FpG7MKcDjY4+uWGMWv3Iyt4HPhDZuEuZZcVBwatjB0SE4JtuQqxuJD3/PAnjHyV/82Z8+vTBp3v3uq62pp1VqhIUFonITiAfuMecOrPgFI6ZLCKRSqlEs3noqFmeAJRsHG5qlpWilJoCTAGIjY2tVkDR6ldhUQL4QHhQyzLLCrZvx6N5c6z+/q6yRtdfhzM/j6MTXwcg6s036qyumna2qsoczU+IyBtApjmHch5G+//JWgjcCkw0/35Xovx+EZkN9DGPl1j+LrQzUZFKx6IgxKvsNJn527fjE1P2KiBk9GjEYqFw/358+vSpi2pq2lnthEFBRHyAe4FojGabKKAdVbhnwZyneTAQKiLxwPMYwWCuOWfzQYz7HwAWA5cCe4A84LZqnot2GssusFHklksgvlgt1lLL7MeOYT+SiNfN5Y9fCB41qi6qqGkaVWs++hzYCJxrvk7AmK/5hEGhgnmaAYaUs64C7qtCfbQzUHxyKtlWB8GWsumxXf0JevpMTat3VRl91Fop9QZgA1BK5VF6qk5NO6H0+F0cdbMS5h1aZtm/QaFjXVdL07TjVCUoFImIN6AARKQ1UFirtdIanLzEXRy1WokKbEbis8+RPmeua1nB9u24N48u1cmsaVr9qEpQeB74EWgmIjMx7kJ+rFZrpTU4RWn/kGW10qRRKzK+/Zbk11/HdtQYeJa/fTveuulI004LVUlzsRwjQ+poYBYQq5RaVbvV0hqavNx9ADQt9Ae7HZWXR8qkSdjT041OZh0UNO20UNXpOL2AdHP9jiJSnMJC08pQSpFX5CA5Jwt38UREcNiMW07C0hwAeHXrSub8b/Bs0cJ4rYOCpp0WqjIk9XXgemA7UJyhTAE6KGhsTTzMLT/chaRej9ia4FSK3IIiulk3sr/lN1yQ7seADH/c/ZOAQAJTC3AAkS+9xKFRt3L0/94FwKuj7mTWtNNBVa4URgLtlFK6c1kr4+O1y3G4x9M1fAa3F8Xg6cyjc+YqZnrnsssayJrgLMapbL73bgQ48U7OJNfDA882bQi9/36SX34Z9+horAEB9X0qmqZRtY7mfYB7bVdEO/MU2BwkH1wKwDZLMuekTuO8o7PwjmrD3LAo2ga1IdsC3w65i6zeN+Pt5g0JibhHN0MsFhpdfx2e7drh26d3PZ+JpmnFqnKlkAdsEZGVlBiKqpQaW2u10s4Ii7cmEuC2m0CnItfNnRlDH+ap3k+yZN9Cjq15lom9HmPeP/OYETeDLmFdCPcJx3boMB7NjESG4u5Oy6/ngltVu7Y0TattVfnfuNB8lKQT0Wn8sGYjOZ65dPJuRmjTPizY+x33xtzHl3Ff0iaoDX0j+9LYpzErDq1gXeI6ejWOpejwFnzPPde1D/HwqMcz0DTteFVpPgpSSn1R8gHoWU3OctsSMmmT9AMH3N1oFdWbUR1HkW/P58nVT7IrfRf/6fAfRIRWQa0Y3nI4AM1tAaiCAtyj9Uxpmna6qkpQuLWcstE1XA/tDDNz7QEGe/1GvsVCq4jutAtux7lR57I6YTVBnkEMbzXcte493e7BTdxoleMDgIeeB0PTTlsVNh+JyI3ATUBLESnZfORP2Sk2q0xE2gFzShS1Ap4DgoA7gRSz/Cml1OKTPY5WsxKyjvHOmm9p6TkQgH1bfsXhdwwIp1VgKwBu7Xgrvx/5nWvPuRYvNy/Xts0CmjH7stkErNxIBgt0UNC001hlfQq/A4lAKPB2ifJs4O+TPaBSahcQAyAiVoysq99ipMr+P6XUWye7b612FNntXPvNfWTLNvIO5OPIb8mrHqvY4+kN4AoK/aL68cEFH9A7suxoonbB7TiasASsVtyjouq0/pqmVV2FQUEpdRBjvoN+tXj8IcBepdTBmpriU6t5/104kWzZBsC4/vsY07QNbgv+5MWwtjSy2mjkZXQxiQiDmg2qcD+2Q4dxj4pC3PUIZ007XVXYpyAiq82/2SKSVeKRLSJZNXT8GzDyKRW7X0T+FpHPRKTczmwRuUtENojIhpSUlPJW0WrQR+sWsSlrLj1tEXQuLOTPvd/jPvt6pCCTfT5+tAwsO7VmRYoOH8ajme5k1rTTWWUdzTcDKKX8lVIBJR7+SqlTvv1URDyAERgT9gBMBlpjNC0lUrrJykUpNUUpFauUig0LCzvVamiV2Jiwl8nbX8LHEcpHSVvo49OMrd6+5I5ehBrzO/sK02gd1LrcbZVSZHzzLbbko66yokOH9MgjTTvNVRYUvi1+IiLza+HYlwCblFLJAEqpZKWUQynlBD4B9G2u9SizII+7lj4I4mQG7vhYvenbbzx25WCjm+JYYASZhZmu/oTjZS9dRuJTT3H0baOLyJGZiTMzE4/o5nV5GpqmVVNlQaFkI3/5//NPzY2UaDoSkcgSy64EttXCMbUqunH+4xRZD/JQo6G0O7wWhjxLTPPz8bB4sDZxLfsyjVTY5QUFZ14eyRMngghZi5dgS06m6NBhADz0lYKmndYqCwqqguenTER8gQuBb0oUvyEiW0Xkb+B84KGaPKZWdY8v/YTD9lXEeA/n9t3zIKo7xN6Ol5sX3Rt3N4JChhEUWga0IGP+fOwl+ndSJ3+MPSmJqNcngsNB+syvKDp0EAB3PRxV005rlQ1J7WZ2KAvgXaJzWQB1Kv0KSqlcIOS4sltOdn9azfl+x3p+OPIh0Y4IPk9dAHlpcPNcsFgB6BvZl/c2vceG5A34uPngv/UAh59+BmtQEBETXsSzTVvSpk0jcORIAkeMIHv5CjLmzCHohhsAdEezpp3mKhuSaq3Liminhw/X/R/+ysFXRzbgFhED135hXCmY+kT0AWDFwRW0C25HwfbtALhFRpIw9kGsISFYvLwIH/8IAMG3jSZ7+XLSZ87ELSwMi7d3nZ+TpmlVV5U0F9pZ4lBGCkls47LcfIKumAz/XQnRfUqt0zGkI/7u/tiVndZBrSmIi8O9SRNazp1D6L334MjIIPzhh3ALDQXAu3t3vLp2xZmTg3tz3XSkaac7HRQ0lw9Wf4FD4ILw86HbDWAp+8/DarHSK6IXAC0DW1KwPQ6vTp0Qd3fCxo6l3Z/raXTjja71RYSQ0Ub6rOKU2Zqmnb50UNBcNictoE1REX0Gjat0vT6RxtVDa2sEtkOHSk2lafHxKbO+/0UX4du/P36DKr7bWdO004Oe3UQD4Nd920hyz+Su3AAsEZ3KLHcWFpK/cSO+557LpS0vZV/mPrqm+3MU8OpUdv2SxM2N6Kmf1lLNNU2rSfpKQQPgy7UfYFGK4Z3LDgJTSpH49DMcuv0O8jZtJsgriGf6PgO7jGGpXh071HV1NU2rJTooaNgdDuLy19E3v4hWfW4vszz9q6/IWrQIgOylS13lBdu34xYRgVtISJltNE07M+mgoDHjz+/JdLPT36sjePqVWpa/ZQvJE1/Hb/Bg/AYNImv5MpQy7mUsiIs7YdORpmlnFh0UNJZu/wQ/p5MrBj7iKlNKkb9tO/EPjsM9IoKo1yfif/HF2I8kUrBtG46cXIoOHNBNR5rWwOiO5rPc3rQkdlkPMTzPSmCr87AdPUraxx+T/dPP2JOSEB8fWsz8EmtgIP4XnE+imxvZy5bhN3AgKKWvFDStgdFB4Sz33k/vYhcY0fwKECH5pZfIWfULvoMG4j92LH7nD8atkTG1hTUwEN8+fchatgyr2Y9QcjiqpmlnvnoLCiJyAGNqTwdgV0rFikgwxvzNLYADwHVKqfT6qmNDZ3c42Jy1nJ62QnoPGkvhvv1kr1hJyN13ET5uXLnb+F90EUnPP0/mdwtxCwvDPTy8biutaVqtqu8+hfOVUjFKqVjz9RPASqVUW2Cl+VqrJf9b/z0ZbkUMszYHvzDSPpuKeHgQfEvFuQn9hw4Bi4XCHTv0VYKmNUD1HRSOdwXwhfn8C2Bk/VWl4Vsc9wkhdgdX9LsPW3Iymd8tJOjqqyodYuoWEoJPrBHDdX+CpjU89RkUFLBMRDaKyF1mWWOlVKL5PAlofPxGeo7mmrH5yH4OWw9xWZ4dnw7DOfbFdHA4CL7tthNu63/RRQB4ddJXCprW0NRnR/N5SqkEEQkHlovIzpILlVJKRMpM7qOUmgJMAYiNja3RyX/OFkopJv78IaAY0fRiHDl5ZMyeTcAll1RpvoOgK0fizMvDd8CA2q+spml1qt6uFJRSCebfoxjzQfcGkoun5TT/Hq14D9rJcDoVzyzcwO7CFVyQl885/e8hfc5cnHl5hPz3jirtw+LrS+hdd2Lx8Kjl2mqaVtfqJSiIiK+I+Bc/By7CmJN5IXCrudqtwHf1Ub+GyuZwMv7rv1jyz2fYrA7u9GqBCjmH9Nmz8OnbF68O+kY0TTvb1VfzUWPgWxEprsNXSqkfReRPYK6I3AEcBK6rp/qdsTIL8hg++y6yMhtjyRxWapndoci1ZRPZdg398ovodO0Usn/5FfuRRBo/rgd6aZpWT0FBKbUP6FZOeRowpO5r1DA4nU6umzeeTPkLgqBvkx40di/9NnvlTuQrm2JMp9sgpDXps1/HLSwM/wvOr59Ka5p2WtF3NDcgDy75gCOO34jxvZxsy1/8UzSFiQM+IMjNBxw28o7t4ZLfN9HfLZAu5z1OUXw8ub+tJvSeMYi7e31XX9O004AOCg3E5xuX83PKVKKdrZi2bw677ZncGBXBi/NH8s7RVNItFj4JCuBYYABjBr0GImTMmQMiBF17bX1XX9O004QOCg3ApoOpvLPlBYKVN3OOrMYa3Ir2/Z7jgdQN/F/SKq7r2It/8o/iRDE06jximg/GWVRExvxv8Dv/fNwjI+v7FDRNO03ooHCGW7svjfFz/gdRWTyfnIL/ORfDyMngFcCtzv+wedU4EnIS+G/bKxgSPYQOwcYIo+wff8Rx7BiNbrihns9A07TTiQ4Kpwmn08mfh/fja2lU5W0OJSaSsOg1Lgj7gx+dXvTv8xAMeBQsxkhjq8XK+xe8X/o4+fmkTP6YtM8/x6N1a3z7n1uj56Fp2plNB4XTwI7kwzz5/Sj2uqfy5tFULs7Nq9J2nYEiizA0oCWDmpyL16DHK10/948/SHzmWWwJCQRecQXhjz2KWE639FeaptUnHRTq2Vu/zuLrPRNxWu00cXjwbHgEXl6DaWrxP+G2YrFwpO05pG9+jYvajnSVZ//0E9ZGjfDp3t1VlvXjjySMfxSPZs2Inv4Fvr1718bpaJp2htNBoZ44nIr7533A6vwpdLIX8nj4pUQOeZzrfrie97wOMPPSmfi4+5Taxu6082XclwxpPoRm/kaOollrX8bbzZvzmpyHsttJfuMN0qfPACDo2msIHz+e7J9+JvHpp/GOiaHZ/z7G6n/igKNp2tlJtx3Ug7wiOw9M/42DGZ/QpsjG5z2fpvvId4nwj2TiwInszdjLy2tfRqnS+f5m7pjJ2xvf5t4V95JdlI3D6WDFwRWc1+Q8PPLtHB5zD+nTZ9DollsIvuN2Mr75lr0XDSPxySfx7duH6E8/0QFB07RK6SuFWrJs9xY2xyfRxLtsPqFv/9zLpTlP8FuUYkKL6/Dueatr2blR53JPt3v46K+PaBfcjls7GcsSchL4cMuHdAjuwO703Tz525Pc2ulW0grSuKjJBRy8ZRSFe/YQMeFFGl1nZAcJvOwykl55BbewMKImTsTi6Vk3J69p2hlLB4Ua9vO+rby0+l1S1HpQFoIOXId74b9TVgqKpz3m8G3jNPofCqLz9DXsVhdg8fTCLaIxURNf5+5ud7M7YzdvbXiLxj6NGdZiGK+sfQWA985/j1Xxq3h13avEpcXhafUk5rck0nfupMmk9wgw5zoA8OrQgRZfflnn74GmaWeuOg8KItIMmI6RFE8BU5RS74nIC8CdQPHMOU8ppRbXdf1OhlKKTYcyePWXt/nH+R2eSrgtO5fFvp54NZ3B3IQkfEo0BR10c+MPj0g+WWpFvCz49IpFFRWSvfInkiZMoOkH7/PagNdIy0/jqdVPsTV1K78l/MbjncbifO5NLr/qana0uZJv93zLpSEDyHzpU3z798f/wgvr8V3QNK0hqI8rBTvwiFJqk5k+e6OILDeX/Z9S6q16qBMAToeD9799glbZToLwrtpGykle8h58bNvZHy0Mzsvn+TwLoe1HMiCoMXfs/ZLXe47gxeYjXJt8mfQLV89ehVdqNlEzPsSnVy8A0qZO5eibb5G9bDkBwy5i0gWTGLVkFNPjptMhuANDlqeQsXgJ2T/9zPhP/4dfRz8u/z4FZ3Y24Y89hpl1VtM07aTVeVAwp9tMNJ9ni8gOoEld16M8H8x9hE+LVtLOVsSMxGS8VdUmdsvGl+eaRmOxFPD00PcJbXMxWCz0Au7w8eLTrZ/Svf1V9I3si81hY+3PE3ltnZOAEZe7AgJA8K23kvnDDyS9/BK+ffsQGBjI5KGTeWvDW9ztezEZTzxMwKWXUhAXR+K9DzDmpZdI+OYRgq65Bq9259TSu6Jp2tlEjh/hUqcHF2kB/IpxH9bDwGggC9iAcTWRXtn2sbGxasOGDTVSl7+2beCp1bdw9QphZWcLTQdfzMR+L5b59Z2Ul8yKwz/TOrAV/SKMsf7/ZB/mmkXXMrrTaB6OfRgAe3o6ztw8iAjl1h9Hsy1tm7EDpXhqjpOYo960/fFH3MLCSu0/f/t2Dlx3PUFXXUnkSy8ZmzidHPzPLRTt30+rxT+g8vI4cNPN2JOTsfj60nrpj7iFhtbI+6Bp2lmhwmaFeutoFhE/YD4wTimVJSKTgZcw+hleAt4Gbi9nu7uAuwCio6NrpC5ZeQUsXHEXPXZY6LfNSa+9Vh7z+5Evw7txS8dbSMxJ5Pcjv7PkwBLWJ67HJ9+JzQ3+2+s+7u56N+9v+QA/Dz9uzuzA4Xvvo2DHDuyJiUZ9fXx4qVULjgW0h8JCLDn5+O6Pp/FTD5YJCADenToRctto0j6diiM7h0Y33ogtPp78TZuIfOVl3Bo1gkaNiJ76KYfuuouQO+7QAUHTtBpTL1cKIuIOLAKWKqXeKWd5C2CRUqpzZfs52SuFo5lZvL3mE7oE9MfXGkDm+knMtPzKW59AUI8+FO7ZQ7rKZfwtDgLCm3Io+xAATX2bcNeBVrT/ai3pjdx5+Npc2jXvyaajm3jG9xq6vTgfa2goPj164NWxAxZfPwr37qVw927sKSlYfHyw+Pjg2bYtjZ94HHErPyY7CwtJmTSJjHnzcWZmggjeMTE0n/llqbQUSindj6Bp2smo8IujzoOCGN9iXwDHlFLjSpRHmv0NiMhDQB+lVKUpPE82KMz/9RNe2D8Jd6UYmptHnsVCx589OD/OSusfFuHIzOTgLaM43NST5bd3oUd4D3r4tsPro9nkrl6Nd0wMBdu3k90ijPuuSCHaGcQr0+xY/f1pOWcO1qCgatepPM6CArIWLyH7p5WEP/QQnq1b18h+NU07651WQeE84DdgK+A0i58CbgRiMJqPDgB3FweJipxsUMg/spc9Tz3Ar52FmWGJBCcX8cbnDkJuHU3jJ4ykcpmLfuDI+PGl6+7tTfj4R2h0443k/PQT8Q+OQ8V0RDKysBw9Ros5s/Fs1ara9dE0Tatjp09QqEknGxTyNm3i8J134czNxa15NIXYcc/Mo82ypVgDAlzr5f7xB0UHD4EIWATffv3waNrUtTxjwQISn3gSrFaaTfkffv3718h5aZqm1TIdFI7nzMsja+kyMubPI3/DRiJefJFG119X7f1k/bgUcXfDf8iQk6qHpmlaPdBBoTKOrKxSVwiapmkNXIVBQWdJBR0QNE3TTDooaJqmaS46KGiapmkuOihomqZpLjooaJqmaS46KGiapmkuZ/SQVBFJAQ5WY5NQILWWqnM6OxvP+2w8Zzg7z/tsPGc4tfNOVUpdXN6CMzooVJeIbFBKxdZ3Pera2XjeZ+M5w9l53mfjOUPtnbduPtI0TdNcdFDQNE3TXM62oDClvitQT87G8z4bzxnOzvM+G88Zaum8z6o+BU3TNK1yZ9uVgqZpmlaJsyYoiMjFIrJLRPaIyBP1XZ/aICLNRORnEYkTke0i8qBZHiwiy0Vkt/m3UX3XtTaIiFVENovIIvN1SxFZZ37mc0TEo77rWJNEJEhE5onIThHZISL9zobPWkQeMv99bxORWSLi1RA/axH5TESOisi2EmXlfr5imGSe/98i0uNkj3tWBAURsQIfApcAHYEbRaRj/daqVtiBR5RSHYG+wH3meT4BrFRKtQVWmq8bogeBHSVevw78n1KqDZAO3FEvtao97wE/KqXaA90wzr1Bf9Yi0gQYC8Sac7hbgRtomJ/1NOD4ewkq+nwvAdqaj7uAySd70LMiKAC9gT1KqX1KqSJgNnBFPdepximlEpVSm8zn2RhfEk0wzvULc7UvgJH1UsFaJCJNgeHAp+ZrAS4A5pmrNKjzFpFAYCAwFUApVaSUyuAs+KwBN8BbRNwAHyCRBvhZK6V+BY4dV1zR53sFMF0Z1gJBIhJ5Msc9W4JCE+BwidfxZlmDJSItgO7AOqBxifmuk4DG9VWvWvQu8Bj/zvsdAmQopezm64b2mbcEUoDPzSazT0XElwb+WSulEoC3gEMYwSAT2EjD/qxLqujzrbHvuLMlKJxVRMQPmA+MU0pllVymjOFmDWrImYhcBhxVSm2s77rUITegBzBZKdUdyOW4pqIG+lk3wvhV3BKIAnwp28RyVqitz/dsCQoJQLMSr5uaZQ2OiLhjBISZSqlvzOLk4ktJ8+/R+qpfLekPjBCRAxhNgxdgtLcHmU0M0PA+83ggXim1znw9DyNINPTPeiiwXymVopSyAd9gfP4N+bMuqaLPt8a+486WoPAn0NYcoeCB0TG1sJ7rVOPMdvSpwA6l1DslFi0EbjWf3wp8V9d1q01KqSeVUk2VUi0wPtuflFI3Az8D15irNajzVkolAYdFpJ1ZNASIo4F/1hjNRn1FxMf891583g32sz5ORZ/vQmCUOQqpL5BZopmpWs6am9dE5FKMdmcr8JlS6pX6rVHNE5HzgN+Arfzbtv4URr/CXCAaI6vsdUqp4zuwGgQRGQyMV0pdJiKtMK4cgoHNwH+UUoX1WL0aJSIxGB3rHsA+4DaMH3oN+rMWkReB6zFG220G/ovRft6gPmsRmQUMxsiGmgw8DyygnM/XDJAfYDSl5QG3KaU2nNRxz5agoGmapp3Y2dJ8pGmaplWBDgqapmmaiw4KmqZpmosOCpqmaZqLDgqapmmaiw4KWp0TESUib5d4PV5EXqihfU8TkWtOvOYpH+daMzPpz+Usaysii0Rkr4hsNDPXDqztOlVEREaWTAApIhNEZGh91Uc7vemgoNWHQuAqEQmt74qUVOKO2Kq4A7hTKXX+cfvwAn4ApiilWiulegIPAK1qrqZlmZmAKzISIzswAEqp55RSK2qzPtqZSwcFrT7YMaYSfOj4Bcf/0heRHPPvYBH5RUS+E5F9IjJRRG4WkfUislVEWpfYzVAR2SAi/5h5kYrnWnhTRP40883fXWK/v4nIQow7Y4+vz43m/reJyOtm2XPAecBUEXnzuE1uBv5QSrnumFdKbVNKTTO39TXz5K83E9ldYZaPFpFvRORHM1f+GyXqcJGI/CEim0TkazO3FSJyQEReF5FNwLUicqd5fn+JyHzzrt9zgRHAmyKyRURal3yPRWSIWY+tZr08S+z7RfOYW0WkvVk+yNzPFnM7/xN92NqZRQcFrb58CNwsRgroquoGjAE6ALcA5yilemPc1ftAifVaYKRLHw58bP56vwPj1v9eQC/gThFpaa7fA3hQKXVOyYOJSBRGnv4LgBigl4iMVEpNADYANyulHj2ujp2ATZWcw9MYaTh6A+djfFn7mstiMO7U7QJcL8akSaHAM8BQpVQP87gPl9hfmlKqh1JqNvCNUqqXUqp4boU7lFK/Y6RAeFQpFaOU2lvi/LwwcvZfr5TqgpFk754S+041jzkZGG+WjQfuU0rFAAOA/ErOVTsD6aCg1Qsze+t0jAlTqupPc86IQmAvsMws34oRCIrNVUo5lVK7MdI/tAcuwsgNswUj7UcIxoQkAOuVUvvLOV4vYJWZfM0OzMSYw6DKRORb8yqjODnhRcATZj1WAV4YKQvAmDwlUylVgHHV0hxjsqSOwBpzm1vN8mJzSjzvbF71bMW4Yul0guq1w0gu94/5+ovjzq+4zhv59/1dA7wjImOBoBLpqrUGojptqJpW097F+FX9eYkyO+aPFRGxYOT1KVYyl42zxGsnpf8tH5+7RQECPKCUWlpygZkrKfdkKl+B7ZT4YlVKXSkisRhzAGDW42ql1K7j6tGH0ufnwDgnAZYrpW6s4Hgl6z4NGKmU+ktERmPkzTkVxfUprgtKqYki8gNwKUagGqaU2nmKx9FOI/pKQas3ZqK2uZSeOvEA0NN8PgJwP4ldXysiFrOfoRWwC1gK3CNGanFE5JwSzTYVWQ8MEpFQsyP3RuCXE2zzFdBfREaUKPMp8Xwp8ICIiFmP7ifY31pzf23M9X1F5JwK1vUHEs1zvLlEeba57Hi7gBbF+8Zokqv0/ESktVJqq1LqdYzsw+1PUH/tDKODglbf3sbIAlnsE4wv4r+Afpzcr/hDGF/oS4AxZnPMpxhNMpvEmAj9f5zgStlMPfwERlrmv4CNSqlKUzIrpfKBy4AxZof4Hxh9Ai+bq7yEEej+FpHt5uvK9pcCjAZmicjfwB9U/EX8LEbT2Bqg5K/32cCjZsewq0PefF9uA742m5ycwMeV1QcYZzaH/Q3YMN5jrQHRWVI1TdM0F32loGmaprnooKBpmqa56KCgaZqmueigoGmaprnooKBpmqa56KCgaZqmueigoGmaprnooKBpmqa5/D8vj3eBPTPoqAAAAABJRU5ErkJggg==", "text/plain": [ "
" ] diff --git a/examples/03_cnn_mnist.ipynb b/examples/03_cnn_mnist.ipynb index f0ffcdf..b7163c0 100755 --- a/examples/03_cnn_mnist.ipynb +++ b/examples/03_cnn_mnist.ipynb @@ -28,14 +28,6 @@ "execution_count": 1, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - ":228: RuntimeWarning: scipy._lib.messagestream.MessageStream size changed, may indicate binary incompatibility. Expected 56 from C header, got 64 from PyObject\n", - "WARNING:absl:No GPU/TPU found, falling back to CPU. (Set TF_CPP_MIN_LOG_LEVEL=0 and rerun for more info.)\n" - ] - }, { "name": "stdout", "output_type": "stream", @@ -192,8 +184,8 @@ "train_evaluator = VisionFitness(\"MNIST\", batch_size=1024, test=False)\n", "test_evaluator = VisionFitness(\"MNIST\", batch_size=10000, test=True, n_devices=1)\n", "\n", - "train_evaluator.set_apply_fn(param_reshaper.vmap_dict, network.apply)\n", - "test_evaluator.set_apply_fn(param_reshaper.vmap_dict, network.apply)" + "train_evaluator.set_apply_fn(network.apply)\n", + "test_evaluator.set_apply_fn(network.apply)" ] }, { @@ -205,26 +197,7 @@ "from evosax import OpenES\n", "strategy = OpenES(popsize=100, num_dims=param_reshaper.total_params, opt_name=\"adam\")\n", "# Update basic parameters of PGPE strategy\n", - "es_params = strategy.default_params.replace(\n", - " sigma_init=0.01, # Initial scale of isotropic Gaussian noise\n", - " sigma_decay=0.999, # Multiplicative decay factor\n", - " sigma_limit=0.01, # Smallest possible scale\n", - " init_min=0.0, # Range of parameter mean initialization - Min\n", - " init_max=0.0, # Range of parameter mean initialization - Max\n", - " clip_min=-10, # Range of parameter proposals - Min\n", - " clip_max=10 # Range of parameter proposals - Max\n", - ")\n", - "\n", - "# Update optimizer-specific parameters of Adam\n", - "es_params = es_params.replace(opt_params=es_params.opt_params.replace(\n", - " lrate_init=0.001, # Initial learning rate\n", - " lrate_decay=0.9999, # Multiplicative decay factor\n", - " lrate_limit=0.0001, # Smallest possible lrate\n", - " beta_1=0.99, # Adam - beta_1\n", - " beta_2=0.999, # Adam - beta_2\n", - " eps=1e-8, # eps constant,\n", - " )\n", - ")" + "es_params = strategy.default_params" ] }, { @@ -235,9 +208,9 @@ "source": [ "from evosax import FitnessShaper\n", "fit_shaper = FitnessShaper(centered_rank=True,\n", - " z_score=True,\n", + " z_score=False,\n", " w_decay=0.1,\n", - " maximize=True)" + " maximize=False)" ] }, { diff --git a/examples/04_lrate_pes.ipynb b/examples/04_lrate_pes.ipynb index 95abf5e..481196d 100755 --- a/examples/04_lrate_pes.ipynb +++ b/examples/04_lrate_pes.ipynb @@ -33,15 +33,7 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - ":228: RuntimeWarning: scipy._lib.messagestream.MessageStream size changed, may indicate binary incompatibility. Expected 56 from C header, got 64 from PyObject\n" - ] - } - ], + "outputs": [], "source": [ "import jax\n", "import jax.numpy as jnp\n", @@ -85,15 +77,18 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 5, "metadata": {}, "outputs": [ { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:absl:No GPU/TPU found, falling back to CPU. (Set TF_CPP_MIN_LOG_LEVEL=0 and rerun for more info.)\n" - ] + "data": { + "text/plain": [ + "EvoParams(opt_params=OptParams(lrate_init=0.05, lrate_decay=1.0, lrate_limit=0.001, momentum=None, beta_1=0.99, beta_2=0.999, beta_3=None, eps=1e-08, max_speed=None), T=100, K=10, sigma_init=0.1, sigma_decay=1.0, sigma_limit=0.01, init_min=0.0, init_max=0.0, clip_min=-3.4028235e+38, clip_max=3.4028235e+38)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" } ], "source": [ @@ -103,7 +98,7 @@ "\n", "strategy = PersistentES(popsize=popsize, num_dims=2)\n", "es_params = strategy.default_params.replace(\n", - " T=100, K=10\n", + " T=100, K=10, sigma_init=0.1\n", ")\n", "\n", "rng = jax.random.PRNGKey(5)\n", @@ -111,7 +106,9 @@ "\n", "# Initialize inner parameters\n", "t = 0\n", - "xs = jnp.ones((popsize, 2)) * jnp.array([1.0, 1.0])" + "xs = jnp.ones((popsize, 2)) * jnp.array([1.0, 1.0])\n", + "\n", + "es_params" ] }, { @@ -123,23 +120,23 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "0 [ 0.01 -0.01] 2423.4482\n", - "500 [ 0.08029711 -0.7827603 ] 2423.3083\n", - "1000 [ 0.17781787 -0.6900547 ] 2423.4324\n", - "1500 [ 1.7823172 -0.5671913] 1363.6532\n", - "2000 [ 2.6007807 -0.43554983] 585.6527\n", - "2500 [ 2.7024412 -0.4344938] 576.2598\n", - "3000 [ 2.737832 -0.47443515] 576.0917\n", - "3500 [ 2.7500708 -0.5119995] 565.1986\n", - "4000 [ 2.750747 -0.51908237] 574.359\n", - "4500 [ 2.765111 -0.5706135] 573.79443\n" + "0 [ 0.05 -0.05] 2423.374\n", + "500 [ 0.13214235 -2.474788 ] 2423.2078\n", + "1000 [ 3.9050057 -4.4652762] 1183.7357\n", + "1500 [ 2.5583386 -4.036586 ] 582.6147\n", + "2000 [ 2.7078283 -3.8439238] 564.5876\n", + "2500 [ 2.744315 -2.5619094] 559.23505\n", + "3000 [ 2.7431633 -3.8979192] 566.58826\n", + "3500 [ 2.7665381 -4.55985 ] 558.7182\n", + "4000 [ 2.7644894 -3.5615964] 556.5793\n", + "4500 [ 2.7446108 -4.953268 ] 559.6667\n" ] } ], @@ -168,13 +165,6 @@ " )\n", " print(i, state.mean, L)\n" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/examples/05_quadratic_pbt.ipynb b/examples/05_quadratic_pbt.ipynb index bf34df8..8e08111 100755 --- a/examples/05_quadratic_pbt.ipynb +++ b/examples/05_quadratic_pbt.ipynb @@ -33,15 +33,7 @@ "cell_type": "code", "execution_count": 1, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - ":228: RuntimeWarning: scipy._lib.messagestream.MessageStream size changed, may indicate binary incompatibility. Expected 56 from C header, got 64 from PyObject\n" - ] - } - ], + "outputs": [], "source": [ "import jax\n", "import jax.numpy as jnp\n", @@ -70,15 +62,7 @@ "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "WARNING:absl:No GPU/TPU found, falling back to CPU. (Set TF_CPP_MIN_LOG_LEVEL=0 and rerun for more info.)\n" - ] - } - ], + "outputs": [], "source": [ "from evosax.strategies import PBT\n", "\n", @@ -123,7 +107,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 4, diff --git a/examples/06_restart_es.ipynb b/examples/06_restart_es.ipynb index 00024c1..39c3308 100644 --- a/examples/06_restart_es.ipynb +++ b/examples/06_restart_es.ipynb @@ -32,7 +32,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 1, "metadata": {}, "outputs": [ { @@ -49,7 +49,7 @@ "\n", "from evosax import OpenES, ParameterReshaper, FitnessShaper, NetworkMapper\n", "from evosax.utils import ESLog\n", - "from evosax.problems import GymFitness\n", + "from evosax.problems import GymnaxFitness\n", "\n", "rng = jax.random.PRNGKey(0)\n", "network = NetworkMapper[\"MLP\"](\n", @@ -71,26 +71,26 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - "evaluator = GymFitness(\"CartPole-v1\", num_env_steps=200, num_rollouts=16)\n", - "evaluator.set_apply_fn(param_reshaper.vmap_dict, network.apply)" + "evaluator = GymnaxFitness(\"CartPole-v1\", num_env_steps=200, num_rollouts=16)\n", + "evaluator.set_apply_fn(network.apply)" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "WrapperParams(strategy_params=EvoParams(opt_params=OptParams(lrate_init=0.01, lrate_decay=0.999, lrate_limit=0.001, momentum=None, beta_1=0.99, beta_2=0.999, eps=1e-08, max_speed=None), sigma_init=0.04, sigma_decay=0.999, sigma_limit=0.01, init_min=0.0, init_max=0.0, clip_min=-3.4028235e+38, clip_max=3.4028235e+38), restart_params=RestartParams(min_num_gens=50, min_fitness_spread=1e-12, popsize_multiplier=2, copy_mean=False))" + "WrapperParams(strategy_params=EvoParams(opt_params=OptParams(lrate_init=0.05, lrate_decay=1.0, lrate_limit=0.001, momentum=None, beta_1=0.99, beta_2=0.999, beta_3=None, eps=1e-08, max_speed=None), sigma_init=0.03, sigma_decay=1.0, sigma_limit=0.01, init_min=0.0, init_max=0.0, clip_min=-3.4028235e+38, clip_max=3.4028235e+38), restart_params=RestartParams(min_num_gens=50, min_fitness_spread=1e-12, popsize_multiplier=2, copy_mean=False))" ] }, - "execution_count": 12, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -113,16 +113,16 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "WrapperParams(strategy_params=EvoParams(opt_params=OptParams(lrate_init=0.01, lrate_decay=0.999, lrate_limit=0.001, momentum=None, beta_1=0.99, beta_2=0.999, eps=1e-08, max_speed=None), sigma_init=0.04, sigma_decay=0.999, sigma_limit=0.01, init_min=0.0, init_max=0.0, clip_min=-3.4028235e+38, clip_max=3.4028235e+38), restart_params=RestartParams(min_num_gens=50, min_fitness_spread=1e-12, popsize_multiplier=2, copy_mean=True))" + "WrapperParams(strategy_params=EvoParams(opt_params=OptParams(lrate_init=0.05, lrate_decay=1.0, lrate_limit=0.001, momentum=None, beta_1=0.99, beta_2=0.999, beta_3=None, eps=1e-08, max_speed=None), sigma_init=0.03, sigma_decay=1.0, sigma_limit=0.01, init_min=0.0, init_max=0.0, clip_min=-3.4028235e+38, clip_max=3.4028235e+38), restart_params=RestartParams(min_num_gens=50, min_fitness_spread=1e-12, popsize_multiplier=2, copy_mean=True))" ] }, - "execution_count": 13, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -135,29 +135,39 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 5, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/rob/anaconda3/envs/mle-toolbox/lib/python3.9/site-packages/flax/core/scope.py:740: FutureWarning: jax.tree_leaves is deprecated, and will be removed in a future release. Use jax.tree_util.tree_leaves instead.\n", + " abs_value_flat = jax.tree_leaves(abs_value)\n", + "/Users/rob/anaconda3/envs/mle-toolbox/lib/python3.9/site-packages/flax/core/scope.py:741: FutureWarning: jax.tree_leaves is deprecated, and will be removed in a future release. Use jax.tree_util.tree_leaves instead.\n", + " value_flat = jax.tree_leaves(value)\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "Generation: 0 Perf (Best): 25.3125 Perf (Mean): 22.210625 0.89572144\n", - "Generation: 20 Perf (Best): 28.0625 Perf (Mean): 19.783125 0.8995422\n", - "Generation: 40 Perf (Best): 28.0625 Perf (Mean): 21.81625 0.46944278\n", - "--> Restarted Strategy: Gen 50\n", + "Generation: 0 Perf (Best): 23.25 Perf (Mean): 21.77375 0.7692213\n", + "Generation: 20 Perf (Best): 68.0 Perf (Mean): 59.039997 3.990374\n", + "Generation: 40 Perf (Best): 197.25 Perf (Mean): 185.15125 9.462757\n", + "--> Restarted Strategy: Gen 51\n", "--> New Popsize: 200\n", - "Generation: 60 Perf (Best): 30.875 Perf (Mean): 18.859375 0.8144148\n", - "Generation: 80 Perf (Best): 31.875 Perf (Mean): 22.13125 0.83103853\n", - "Generation: 100 Perf (Best): 41.875 Perf (Mean): 24.884687 1.4744185\n", + "Generation: 60 Perf (Best): 200.0 Perf (Mean): 197.47156 3.7782266\n", + "Generation: 80 Perf (Best): 200.0 Perf (Mean): 199.58093 1.214557\n", + "Generation: 100 Perf (Best): 200.0 Perf (Mean): 200.0 0.0\n", "--> Restarted Strategy: Gen 101\n", "--> New Popsize: 400\n", - "Generation: 120 Perf (Best): 61.5 Perf (Mean): 36.045624 5.693629\n", - "Generation: 140 Perf (Best): 108.25 Perf (Mean): 79.3975 9.712329\n", - "Generation: 160 Perf (Best): 176.25 Perf (Mean): 139.6253 18.515297\n", - "Generation: 180 Perf (Best): 200.0 Perf (Mean): 173.19374 9.615619\n", - "--> Restarted Strategy: Gen 189\n", - "--> New Popsize: 800\n" + "Generation: 120 Perf (Best): 200.0 Perf (Mean): 200.0 0.0\n", + "Generation: 140 Perf (Best): 200.0 Perf (Mean): 199.98969 0.20599204\n", + "--> Restarted Strategy: Gen 151\n", + "--> New Popsize: 800\n", + "Generation: 160 Perf (Best): 200.0 Perf (Mean): 200.0 0.0\n", + "Generation: 180 Perf (Best): 200.0 Perf (Mean): 200.0 0.0\n" ] } ], @@ -203,12 +213,12 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAADgCAYAAADsbXoVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABm0UlEQVR4nO2dd3hVRdrAf+8t6b2RhAChhS4tgDRBRQEromJX/FzbWndXV9eya191bWtZu2IXCyqigICAgEjvPYQI6YX0est8f5yTkEASAqbcwPye5z73nDlz5rwzd+55p76vKKXQaDQajQbA0tYCaDQajcZz0EpBo9FoNDVopaDRaDSaGrRS0Gg0Gk0NWiloNBqNpgatFDQajUZTg1YKmhZBRMaLSGpby/FHEZGxIrKrhdJ+REQ+bom0NZrjRSuFdo6IXCkia0WkREQyRGSuiIz5A+kpEelR63y8iLjN9ItFZJeIXN880jcowwxTjgsPC3/RDJ9unk8XkeUNpLFERCpMuXNFZJaIxDQQt5+I/CQiB0WkQETWicg5AEqpZUqpXs2cxWPmsPxUf76vdf0BEdlnhqeKyMwWlOUREXGYzyoQkV9FZGQzpFun7h1nGktE5E9/VJaTGa0U2jEi8lfgJeApoAPQGfgfcGEjtzWUlq2Ry+lKqQAgCLgPeFtE+h6zwMfGbuDaw+SbBuw9hjRuN+VOAEKAFxuI9z2wAIgGooA7gaJjF7nFuV0pFVDrcz6AiFwHXANMMPObCCxqYVlmms+KABYDX7bw8xpFDPT7rBnQhdhOEZFg4DHgNqXULKVUqVLKoZT6Xil1rxlnuIisNFtzGSLyqoh41UpDichtIrIH2CMiv5iXNpmtwMtqP1MZfAvkA31FxFtEXhKRdPPzkoh4NyBvrIh8LSI5Zov2zqNk8XtgjIiEmueTgM1A5rGVFCilDgJfA/3rkSsC6Aq8rZSqMj8rlFLLzet1hsFEJEVE7hWRzSJSKiLvikgHs4dWLCILq2UWkXizjG8yyydDRO5pSE4ROdVsdReIyCYRGd/ELA4D5iul9pr5zVRKvdXAM64/rIexR0S+rHV+QEQGNfG5KKWcwCdARxGJNNMINsslQ0TSROQJEbGa13qIyFIRKTR7cDPN8CPqnoiEisgcs87km8dxtWRdIiJPisgKoAz4CBgLvGqm8aqpLF4UkWwRKRKRLSJyRD3QHEIrhfbLSMAH+KaROC7gLxituZHAmcCfD4szBRgB9FVKnWaGDTRbonWGIETEIiIXYbS6twAPAqcCg4CBwHDgocOFMFtw3wObgI6mHHeLyMRGZK8AvgMuN8+vBT5sJH6DmC/+i4EN9VzOA5KAj0Vkioh0aEKSFwNnYfRAzgfmAg8AkRj/qcMV3ulAT+Bs4D4RmVCPjB2BH4AngDDgHuDr6hftUfgNuNZUVonVL+AGWAqMNX/LWMALo24gIt2AAAzl2yTMRsa1GOWYbwbPAJxAD2AwRr6rh3QeB34CQoE44BWABuqeBXgf6ILRCy4HXj1MhGuAm4BAYDqwjEM9qtvNZ5+G8VsFY/Q285qav5MRrRTaL+FArtlSqxel1Dql1G9KKadSKgV4Exh3WLR/K6UOKqXKG3lWrIgUALnAv4BrlFK7gKuAx5RS2UqpHOBRjD/p4QwDIpVSj5kt8WTgbQ698BviQ4yXXYgp97dHiX84L5tybwIygL8eHkEZxr9OB1KA54EMEflFRHo2ku4rSqkspVQaxktolVJqg1KqAkNJDz4s/qNmT24LxkvuinrSvBr4USn1o1LKrZRaAKwFzjk8P7U+j5t5+Bi4A5iI8dLPFpH76hPcLPtiDEV+GjAfSBeR3hhlvEwp5W4k79VMM8u2HLgRuEQp5TSV6jnA3WaeszGG7ap/awfGSz5WKVVR3SNrQNY8pdTXSqkypVQx8CRH1t8ZSqltZh131JOMA0Nh9AZEKbVDKZXRhPydtDQ2jqzxbPKACBGxNaQYRCQBeAFjjNkP4/ded1i0A014VrpSKq6e8Fjg91rnv5thh9OFQ4qlGivGC7VBlFLLzZbyg8AcpVS5iDRB3BruVEq9c7RISqlU4HYAEekEvIWhkBqaPM2qdVxez3nAYfFrl/HvwIB60uwCXCoi59cKs2OM11fTYH6UUp8An4iIHaP394mIbFRKza8n+lJgPEZLfilQgPGyHWmeN4UvlFJXm72wr4GhwBIzH3YM5Vod18KhMvg7Rm9htYjkA88rpd6r7wEi4oehUCZh9CwAAkXEqpRymeeN1l+l1M8i8irwGtBFRGYB9yilPHHOyCPQPYX2y0qgEuMF0BCvAzuBnkqpIIwhjsPfqn/ETG46xkugms5m2OEcAPYppUJqfQKVUufUE/dwPgb+xnEOHR0rSqkDGC+Q5hx37lTruLEy+uiwMvJXSj19LA8y55W+xBgCaigP1UphrHm8FEMpjKPpSqH6ebkYwzePiLG66wBGvYyolY8gpVQ/M36mUupGpVQscDPwP2l4xdHfgF7ACLP+Vg8x1a7Dh9ffI+qzUuplpdRQoC/GMNK9x5LHkw2tFNopSqlC4J/Aa+ZYuJ+I2EVksog8a0YLxFhFU2IOD9zahKSzgG5NFOMz4CERiTRbjP/EeIkfzmqgWETuExFfEbGKSH8RGdaEZ7yMMX7/SwPXRUR8an+aKHv1zaEi8qg5AWox8/F/GOP0zcXD5u/TD7geqG+56MfA+SIy0SwfHzEmuevroR2eh+kicq6IBJp5mAz0A1Y1cMtSjCEzX7OXtAyjNR5O/fMujWIOJc4H/m4OzfwEPC8iQaY83UVknCnrpbXylI/xEq8erjq87gVi9LwKRCQMY+jyaNRJQ0SGicgIswdVijFX1ZThsZMWrRTaMUqp5zHGyR8CcjBaabdzaOz9HuBKjDHkt6n/ZXQ4jwAfmGPW044S9wmMce/NGBPP682ww+V0AedhjGPvw5ibeAdj4q9RzPmORebYf32Mwnhx1Hyk8eW1h1MFxAMLMRToVoyW7vRjSONoLMWYzF4EPKeU+unwCGYP5UKM3lz1b3kvdf+j1atqqj/VQ4FF5n37MYaCngVubWi8Xim1GyjBHL4zh1KSgRXVwzJm+mOPIY//AW4SkSiMiWcvYDvGi/8roHqPyDBglYiUALOBu8x5Djiy7r0E+GLUl9+AeU2Q47/AJeZqpZcxllG/bcrxO8aw63+OIV8nHdLwf02j0fwRRCQeQwnaG1sQoNF4ErqnoNFoNJoatFLQaDQaTQ16+Eij0Wg0Neiegkaj0Whq0EpBo9FoNDW06x3NkyZNUvPmNWWVmuZ4mTFjBgDTp09vUznaG7rcNA3hIXWjQdMA7bqnkJub29YiaDQazQlFu1YKGo1Go2letFLQaDQaTQ0ttiTVtDb5IYZHMAW8pZT6r2nDZCaGaYEUYJpSKl8Mk4r/xTC7WwZMV0qtb+wZiYmJau3atS0iv0aj0ZzANDin0JITzU7gb0qp9SISCKwTkQUYNmUWKaWeFpH7gfsxXDxOxnBE0hPD6cvr5vcx4XA4SE1NpaKiopmyoWkMHx8f4uLisNvtbS2KRqNpBlpMKZjWEjPM42IR2YHhdetCDLO9AB9g2GC/zwz/0DR89puIhIhIzLE6xEhNTSUwMJD4+HiO0fa+ph5KSkoACAg43EUAKKXIy8sjNTWVrl27trZoHs2vv/4KwKhRo9pYEk1zUeFw8dG6X1mUOh+XuwonDoKssfTxO5+Djn3sLp+Hu9oAq6r1VWswRgEHd2cAirCEGHruTyVhXzYWt8Jpd2NzufArceO2GnF9KhQWxWEGwY0TyykDueThD5o9n62yJNU0DDYYw5Rvh1ov+kyM4SUwFEZthxmpZlgdpSAiN2HYb6dz585HPKuiokIrhGakusdVn1IQEcLDw8nJyWltsTye3bt3A1optAeS8jJ5YOEbdLacj+UwA7sVDje/5X9MiTuLyvQr8Yp7C6tfMri9AUGs5fy2PRBb5BzEOw1x+eNDJd44sIkLNxaqsFGGL8ocsemzIBn/Sug+OIwpy51YDzPk7bSAxQwr9wGXOfOrar3SFPB7aEqLlEeLKwURCcDwzHS3Uqqo9staKaVE5JgmNZThkPwtMOYUGnjm8QusOSZ0WWvaO3f8+AKp7rmk5HbEWtUNhaIq8AdE2fApm4gj+leslHDpqCuYm7eP6/vfwN1D76LMUcakryfh1/db0krSeLjP9Uz75Q0oyYLAGIhIgLI8nPu3k7E+Gq8hZ+I/qDcv730SgIuKnfj3DCP2jY+xBIXhrnSA1Yo1JMQQTCnE0vBaoIbcAv5RWnT1kenY4mvgE6XULDM4y/TQhPmdbYanUddDVZwZ1q7Iy8tj0KBBDBo0iOjoaDp27FhzXlVVdczp7dy5k5EjR+Lt7c1zzz3XAhJrNCcnbrdiye40DlQZzuYemtKBlf84k/suKcYRuBB3yEIeu0JwYgyh7nK9gxs3Z3WZAICf3Y/p/aeTVpJGtE84Uxa9CBY76prvKB3xLvt+8GXfgmj2rehHyX44+M0iUh99FZsfdLj/73R68Sk6fbMUW8euWAKDsUVEYAsNRUSMTyMKoSVpsZ6CuZroXWCHUuqFWpdmA9cBT5vf39UKv11EPseYYC5sjw62w8PD2bhxIwCPPPIIAQEB3HPPPcedXlhYGC+//DLffvtt8wio0Wh4bPHHfJH8Gs7SbtiDywA4UHyA9JJ0nlr1FAmhCezO380Tvz2BTWz0CO3BzoM76eDXgb7hfWvSubzX5fyU8hPXllsp3ZlMWkkCVR//DXdZGfZOnbDHdcSKIu65Zyh58z7yVuYSPOl0rMGhBEy+qK2y3ygtqYpGA9cAZ4jIRvNzDoYyOEtE9gATzHOAHzG8PyVheEr6cwvK1qosWrSIwYMHM2DAAP7v//6PyspKAOLj4/n73//OgAEDGD58OElJSUfcGxUVxbBhw9psdU91q0VzbNhsNmy2dm1F5oRlQ/o+vkz5LxarA3vwZjoFdCXaP5oDxQdYnbmaClcFz572LAMiBpBbnsvQ6KFcmnApAGd0PqPO/8HP7sfn533O8MXpZK7yRzkVwZdcTPQjj9Btzvd0ef99us2ahe/w0US+tZiEuR/jP/Icj64bLbn6aDkNr4U9s574CritOWV49PttbE8vas4k6RsbxL/O79fk+BUVFUyfPp1FixaRkJDAtddey+uvv87dd98NQHBwMFu2bOHDDz/k7rvvZs6cOc0q7x8lPDy8rUVol1x99dVtLYKmAW7/6UEUbt498zOyHbvpGtyVF9a+QGpxKrt9duNr8yU+KJ6Lel7EltwtnN7pdCZ1ncTC3xdyScIl9abpyMrF6mej67ffNNyIstqxdEnk6i6JLZi7P47e0dzCuFwuunbtSkJCAgDXXXcdv/xyyAf9FVdcUfO9cuXKNpFRozlZWJq0jyLZwoiwixjeKYHzup1Hv/B+dArsxIHiA+zO302PkB5YLVbO73Y+tw26jQu6X0CQVxBvnf0WCaEJRybqduEsLMUW7HdC9Ko9tw/TDBxLi76tqF2JPLFCFRcXAxAYGNjGkrQvli41Ji/HjRvXxpJoavPqSsOq8o2Jk+uExwXGcbDiINtytzExfiKseRcft5NbRtxy9ESL0nGVC9YOIU2SwdPrxgmtFDwBq9VKSkoKSUlJ9OjRg48++qhOZZg5cyb3338/M2fOZOTIllpkdvxUz39opXBs7Nu3D/DcP/6JzM7sNH7YsZ0Yn151wiscLrbkrcMvzJfEmIF1rsUFxgFQ4iihZ2hPWPYOuB0w4uajPzB/H84KC76RUU2Sz9PrhlYKLYyPjw/vv/8+l156KU6nk2HDhnHLLYdaH/n5+Zxyyil4e3vz2WefHXF/ZmYmiYmJFBUVYbFYeOmll9i+fTtBQUGtmQ2Npt1w36KX2FuxkJI9D5ubzA4R0H0vQ6KGYrPUffV1Cji0Gj4hNAFKskEdtqusIQ4m46q0YIvpdPS47QCtFFqQRx55pOZ4w4YN9ca59957eeaZZxpMIzo6mtTU1OYWTaM5Yckuz0CsTp69xotxHQ+tackuy+TK+bmc1unIHnl1TwEgIaSnsQFNLKAUHGVY1525B7fTgjWmS/Nlog3RE80ajeaEotSVB8C6nGV0CPKp+SSXbAJgRMyRdjaDvYMJ8goiyi+KYLfbGDpyVUJVSYPPUQ4HrsJCnKnGUnJbRGQL5Kb10T2FNiQlJaWtRTgqljbaVdne8fX1bWsRTkoKyxy4LAVYgGWpy9h1cBdpJWmc0fkMVmeuJtQ71JgzqIfeYb0J8Q4xho6qKcsD71rzaS4HOCvBO4CsZ56leMECOo4rB8AW0bTl255eN7RS0DRKWFhYW4vQLrnsssvaWoSTkl3Z+VhsJcT59SS1bA+XfG/sK5g7dS6/ZfzGsOhhWKT+hs5/T/+vce3AmkOBpXkQGn/ofMG/UDvn4J6+hIJZs1BlZVSkVAA+WMMjmiSjp9cNrRQ0Gs0Jw+YMw9DyOV0vYEHqLOKD4ll8YDFvb3mb7LLsukNHSsHHU8HuBxe9SYC3aQn48J5CLXK/XkzR5jICsh9ElRnmMcrSDbucTe0peDp6bEDTKEVFRRQVNe+u8JOBhQsXsnDhwrYW46RjR46hFE7p0J3ZU2bz8hkvMyBiALP2GPY46yiF1LWw92fYOQdmnAsO0zFXSdahOGW5VOzYQelvqwAo3ZtPZaGdvFlL8OpsTE6X5RnKxNrEXrWn1w2tFDSNUlVVdVzWXU92UlNT9aqxNmBfvmFYOTYguiZsYvxEADr4daBzYC0fLOveB68AmPhvyNgIaeuM8JIsaiz0lOWR/eKLpD/wD6gqparAhU+EwubnJKrbbqw+blzlLixBQVi8vJoko6fXDa0UmpnmNp0NMH78eHr16lWTTnZ29hFxZsyYgYjUaYF8++23iAhfffXVcedHo2lPZJRmAtDBv0NNWLVSGBEz4pDVgPJ81JZZVERMotwRj7NSIN/YVEZJNgTHgcUOpbk4UtNwpmfgTN6Is8xKwJiR9LxCEThuFN79BgFgO4FshOk5hWamuU1nV/PJJ5+QmNi4Ia0BAwbw+eefM2GCYe/9s88+Y+DAgY3eo9G0N0oqK/jznP8SZz0Tm/gAoJQbJ5UUVObiE+BNoP3QiqFo/2ieH/c8fcL71IRVzf4PB2YHUFX8K/ArSDQ9Bm/FPhijpxDQAVwOVGkujvR047kL5gGCV79EuPo9EMF775OUrduslYLm2Fi0aBH33HNPzY7m119/HW9vb+Lj45k2bRpz587F19eXTz/9lB49ehz3c8aOHcuyZctwOBxUVlaSlJTEoEGDaq6vW7eOv/71r5SUlBAREcGMGTOIiYnh7bff5q233qKqqqrGFIefnx/Tp0/HbrezadMmcnNzefbZZ7nkkvqtRGo0rcV/V/zIhpKP2ZRjwVI2ECUVuDq8CZZSvHxjCfeJOsKO2NnxZx86OZhMwRefUFXqR/Sjj6IqK8h66t9U7tmFHaAkm5K8MFRREL4h2SjTJW3xCmNVklffxJoNbd49jeWt1oimrTxqD5zYSmHu/ZC5pXnTjB4Ak58+ejyT5jKdff3112O1Wrn44ot56KGH6jWeJyJMmDCB+fPnU1hYyAUXXFBjZ8XhcHDHHXfw3XffERkZycyZM3nwwQd57733mDp1KjfeeCMADz30EO+++y533HEHANnZ2cydO5esrCwuuOACrRSaiDZD0nL8tHcDeMHfJnfgqj5ncsNPN7Auaz8ANu9i4oMHN57A3Pspz/XCp1cvQi+bhiMrm6yn/l3TI6Aki+zFTlRFJbHBh4ZqS7cZk9hePXrXhHn3NBpxx9JT8PS6oecUWpjmMJ39ySefsGXLFpYtW8ayZcv46KOPGnze5Zdfzueff87nn39ekzbArl272Lp1K2eddRaDBg3iiSeeqJns2rp1K2PHjmXAgAF88sknbNu2rea+adOmER4eTt++fcnKyjrieZr6mTp1KlOnTm1rMU449mQVk1WRDEBOeQ6783ezLmsd9yTeQ7hPOFXuqjrzCUdQcAC1az7l+T74Jg4HwBYZgVgFR1aeMWRUkkdVdilV+Q6qMvIBEKsb5XBj9bVgDQ6uSc67Rw+w2bDHRNf7uPrw9LpxYvcUjqFF31Ycbjrb5XIxdOhQAC644AIee+wxOnbsCBiWSq+88kpWr17NtddeW296w4cPZ8uWLfj5+dUoIgClFP369atX8UyfPp1vv/2WgQMHMmPGDJYsWVJzzdvbu04aGk1bkFVSxG1znqcy7zSsPkaLPqssi/QS43hY9DDyK/J5d+u7dPBrRCls/46KAjuqyonf4EEAiMWCLTyIqsJMyEuiqsSKchrG8EqSigBf/KKqKM3wwR7hXyc5a1AQ8Z99ilfXbs2e57ZC9xRamNqms4F6TWdXf48cORKr1crGjRvZuHEjjz32GE6nk9zcXMAYApozZw79+/dv9JlPP/00Tz31VJ2wXr16kZOTU6MUHA5HTY+guLiYmJgYHA4Hn3zySZ37ysrKKCws/AMlcHIyb9485s2b19ZinDB8sP4ndlXOIs21EIuXsaEspyyH9FJDKcT6xzK151RsFhvxwfENJ7TtG8qrjOu+gw8NM3nFROEotcL+36gsPNRWLsnwxmJ349cz1ogXe6R5bN8BA7AG+B8R3hCeXjdO7J6CB/BHTWdXVlYyceJEHA4HLpeLCRMm1Iz/N8TkyZOPCPPy8uKrr77izjvvpLCwEKfTyd13302/fv14/PHHGTFiBJGRkYwYMaLGsQ6A0+nE4XD8gRI4OcnMzGxrEU4okvON8vSOWILDoQi0B5Jdlk16STq+Nl+CvYMJ8Qlh7tS5RPo2YJiuYD+kraW8dCy26FLsMTE1l+xxnanYuRPWzaCywA4iiM2Cuwq8Q914T7wZfvl3nfmE48XT64ZWCi1Ic5jO9vf3Z926dUd91vTp05k+ffoR4TNmzKg5HjRoUJ35jGpuvfVWbr311nrvre6lAJSUNGwxUqNpSdKKjPmsEodRB0d1HMXSA0tJL0kn1j+2Zhg22r+Rsf0dc1BuKNtXgG/isDqX7F0TcFUuwn1gE5Xu3nh1DsFidVCRnI49MhifUWeB7T/4nnZ+y2TQg9DDRxqNxuPJKc/ForyxipUQ7xAGRAygwlXBrvxdxATEHD0BgL2LKMrvijM3j+Dzzq1zyd6lKwCOUiuVJf54J/TEu3u8cS2uM/aYGBJWLMf/tNOaM1seiVYKbUhKSgoRJ9D6Zo2mJXC7FcXOPIJs0UzuOpkxHcfUTCanlaTRMaDj0RNxVKD2rSBvqxdePboTcMYZdS7bzcUclR0mU5WWiXfPBLwHGAs+7H1PBcAaHOyRftSbGz18pGkUm01XkeMh/ATa4drWpBWUoyzFhPlE8e+x/wZgfdb6musx/k3oKRz4jdIDbioziol5+gHkMD8h1UqhJD8O3OvwTuiJNSTEuNbzlObJiImn1w39j9c0Soj5x9AcG+eff+KPPbcWe3NKEFsxMQGHVt1F+R1aBRQbENvwzVu/hmUvQng3yg/6gAhB55xzRDRbZCTY7RR+9x3W4GD8hg3DGhpKzJNPEHD6+GbMjefXDa0UNBqNR7MnqwixFRMfcqhHUFspNNpTWPUWZG2BrC04LL2wRnjXa81ULBbsMTE4Dhwg9vnna3Yoh1x8cfNlpJ2glYKmUQoKCgDdYzhWvv/+e8DzW4XtgR3ZWYi46RR8aFOal9WLEO8QCioLGu4plGTDgVUw6CrI2oZzRyD2KJ8GnxN+458Qq42AMaObOwt18PS6oSeaW4CsrCyuvPJKunXrxtChQxk5ciTffPNNsz9n586djBw5Em9vb5577rlmTx+MfQpOp7NF0j6RycvLIy8v7+gRNQ0ye8caBrwzge93GoboInzrLsqI8ovCbrEfEV7Dzh8ABSNvg5uX4iyzYItueMlq6KWXEjL1ouYSv0E8vW5opdDMKKWYMmUKp512GsnJyaxbt47PP/+8RZxqhIWF8fLLLzeLaW6NxtP4cvs8sGeR0H0nwBGb0qL9o4kNiG3Q5zI750BoV4jqC4AjOxt7hyN3JGvqopVCM/Pzzz/j5eVVZ9dyly5daqyOulwu7r33XoYNG8Ypp5zCm2++CcCSJUsYP348l1xyCb179+aqq646qq2hqKgohg0bht1ub7kMaTRtxN7i7QDkuI2VRof3CO4achePjXqs/puryiB5KfQ+F0Rwl5fjLizEFtWIXSQNcILPKTyz+hl2HtzZrGn2DuvNfcPva/D6tm3bGDJkSIPX3333XYKDg1mzZg2VlZWMHj2as882bL1v2LCBbdu2ERsby+jRo1mxYgVjxoxpVvk1mvaA0+Wi2J0EVih3lgNHKoWE0IS6NykFK1+DAZdAYSq4HTj8elM+bx4+vQ3zFLYOWikcjRNaKXgCt912G8uXL8fLy4s1a9bw008/sXnz5hoXmYWFhezZswcvLy+GDx9OXJzhDHzQoEGkpKS0uVLQvZDjI7qRsWvN0fklZRtYK4j26U5mxV787f742f0avyk/BX56ECoKINBYkZQ7dzMFs+bQ8aWXALBHt71S8PS60WJKQUTeA84DspVS/c2wR4AbgRwz2gNKqR/Na/8AbgBcwJ1Kqfl/VIbGWvQtRb9+/fj6669rzl977TVyc3NrXGkqpXjllVeYOHFinfuWLFlSx0y11Wr1iAne4Fq24zVNZ9KkSW0tQrtmwd7VAFzdZzrPbXi44cnk2hSa83b7f4Pw7uATQvlaY6Sg2PRd7gk9BU+vGy05pzADqC/3LyqlBpmfaoXQF7gc6Gfe8z8RsbagbC3GGWecQUVFBa+//npNWFlZWc3xxIkTef3112ssj+7evZvS0tJWl1Oj8WQ25WwCly9X9j2PIK+gY1MKqWsgbR3OoL5UmibrS0wfIXpO4ei0WE9BKfWLiMQ3MfqFwOdKqUpgn4gkAcOB+l2ReTAiwrfffstf/vIXnn32WSIjI/H396+xhPqnP/2JlJQUhgwZglKKyMhIvv3220bT/Oc//0liYiIXXHBBnfDMzEwSExMpKirCYrHw0ksvsX379mZ195efb3ieCg0NbbY0TwZmzZoF4NEetjwVpRTpFTsIsvbAbrPxl6F/IcirCXW6Wik4KyBzC+V+FwEpiLc37pISLAEBx+T3oKXw9LrRFnMKt4vItcBa4G9KqXygI/BbrTipZtgRiMhNwE0AnTt3bmFRj4+YmBg+//zzeq9ZLBaeeuqpI5zgjB8/nvHjx9ecv/rqqzXHjz1W/wqL6OjoFlnqWhuXy9Wi6Z+oFBUVtbUI7ZYvNq3FZctidLTx0rwkoRG/4M4qWPIUjLgVCg+A3Q8cRs+8LEsQLy+CJk+m8NtvPWLoCDy/brT2ktTXge7AICADeP5YE1BKvaWUSlRKJUZGNuBMQ6PRtFve3fg1KAt/GX3p0SPvXwnLX4StXxk9hcjeENYdgLKkHHwHDsRvmDGfp/coNI1W7SkopWo8v4vI28Ac8zQN6FQrapwZptFoThKe/uUr0vPcpDlW0ilwADEBTWj0HTAmpMncYiqFXhDdH3dBBhV79hF+0wR8TzGsnNo6ePaqH0+hyT0FETnKerAmpVHbctVFwFbzeDZwuYh4i0hXoCew+o8+T6PRtA8yi/P5OPkJFhc9jsXrIFf1n9K0G1PN10TGZkMpBHeCM/+Fc/J74Hbj1SUer+7dsXfujE//fi0m/4nEUXsKIjIKeAcIADqLyEDgZqXUn49y32fAeCBCRFKBfwHjRWQQoIAU4GYApdQ2EfkC2A44gduUUnow2wPwqseipOboVO830TSNDzcsQMTFyKiJuC0FXJRw2MLFggPw1fVw8TsQGm+Eud1mT0EgezugIDgO/CNwYqxWsneIQiwWus+f5zEOcjy9bjRl+OhFYCJGax6l1CYROapPOqXUFfUEv9tI/CeBJ5sgj6YVac6VTCcTEyZMaGsR2hU/718KLl9envAUPvZ6GiL7VxpLTX97AyY/bYTlJRkb1XpMgCRjHwLBxgvXkWmMVFcbwPMUhQCeXzeaNHyklDpwWJBuxWs0mmbB6XKRVrWBSNsp9SsEMHYrA2z8FKrMfT0HVhnfw/50KJ6pFJxZmYDel3A8NEUpHDCHkJSI2EXkHmBHC8vVrmkt09kzZsxARFho7tYE+PbbbxGRGjMaf5SDBw9y8ODBZknrZGLmzJnMnDmzrcVoF8zZtQasxYztOLbhSPkpYLFBZSFsMet26mrwCYGeZ+O2BOCstBhzCoAjK9tj9iUcjqfXjaYohVuA2zD2DaRhLCe9rQVlate0pulsgAEDBtTZE/HZZ58xcODAZkvf7XbjdrubLb2ThfLycsrLy9tajHbBnN3LAbhm4NkNR8pPgbhhENUPNn5ihB1YDZ2Gg8VK1rZY9s2LRNmN4U5nZiY2D7BzVB+eXjcaVQqmqYn/KqWuUkp1UEpFKaWuVkp5roeINqY1TWcDjB07ltWrV+NwOCgpKSEpKYlBgwbVXF+3bh3jxo1j6NChTJw4kYyMDADefvtthg0bxsCBA7n44otrTHFMnz6dO++8k1GjRtGtWzdmz57dXEWj0dTLnrxURHnTI6IRt5r5KYZvhISJkLYOitIhZyfEDQegPM8bZ7mV4p9/BsCRnYVdDx0dF41ONCulXCLSRUS8lFJVrSVUc5H51FNU7mhe09nefXoT/cADDV5vbdPZIsKECROYP38+hYWFXHDBBezbtw8Ah8PBHXfcwXfffUdkZCQzZ87kwQcf5L333mPq1KnceOONADz00EO8++67NYorIyOD5cuXs3PnTs4999wjzGtoNM1FWZWTnPJsQkPCG47kqDCUQGgXiEuE5S8YJrIBOg3HXV5OZbphjiX/85kETZ6MMzML79E9WiEHJx5NWX2UDKwQkdlAjeU2pdQLLSbVCURrmM6+/PLLefnllyksLOT555+vMaGxa9cutm7dyllnnQUYvZSYGKM1tnXrVh566CEKCgooKSmpY7V1ypQpWCwW+vbtS05OzpEP1GiaiTUp+WAtJNq/kVZ94QFAGUtRO50KYoW174NYoONQKrbvBLcb30GDKFu1iso9e3Dm5GDTO5iPi6Yohb3mxwIEtqw4zUtjLfqWoi1MZw8fPpwtW7bg5+dHQsIhxyNKKfr168fKlUfaFZw+fTrffvstAwcOZMaMGSwxrUgCdeSo71xzdLp27drWIrQLViTlYrEX0T1sQMORzJVHKrgz4h0AsYMhbS1EDwDvACq2GR7aOjz0ECmXXUbu22+D243dQ3cwe3rdOOpEs1LqUaXUoxh2ip6vda6ph7Yynf30008fYWSvV69e5OTk1CgFh8PBtm3bACguLiYmJgaHw8Enn3zSaNqBge2qLeARjBs3jnHjxrW1GB7LxvQU7v7+I37YkobYiogNaOQFnp9CRYGNXRf+mYpdu6kKGMSe7zpQYTG8qVVs3441LAyffn3xHzWKoh/nAnjsRLOn142m7GjuD3wEhJnnucC1SqltLSxbu6Q1TWfXZvLkyUeEeXl58dVXX3HnnXdSWFiI0+nk7rvvpl+/fjz++OOMGDGCyMhIRowYQXFx8R/Kt0ZzLDy1/G12lM+mvOwefMVNlF8jQz35KVQU+qEqKymePx9LmR1nuZWygmB8gIpt2/Dp1w8RIeiccyhdtgwAu4dYRW1vyNFWuIjIr8CDSqnF5vl44Cml1KgWl+4oJCYmqrVr19YJ27FjB3369GkjiU488vKMhWbh4Q1PBOoyP5KPP/4YgKuvvrqNJfFMxn9wPXms5f7h9/P06qd5afxLnNnlzPojf34VOXO3kbuqEp8BA7D4+FC2Zg2hV19F1D33sGtoIuE3/omou+/GVVzMnlGjUQ4HPVf+is0D/YB4SN1ocIt3U+YU/KsVAoBSaomIeN6OEE2L0JRlsZoj8QRXqp5MkSsLrLAibQVA4z2FnF04Kv2BSiq2bgWLMepdtX8/lbt2gcuFT9++AFgDA/E/7TRKV6zAGhLSwrk4Pjy9bjRp9ZGIPIwxhARwNcaKJI1GozlmlFI4xFjVtiZzDdCIUijOgrw9OCqHYQkMxF1cDC4X9thYqlJ+p2K7Mcns2++QBdQO/7ifqn37PMreUXuiKTua/w+IBGYBXwMRZphGo9EcM6mFeWCpAKDCVYFVrA37YE4x5gccRU78x4zGGh6OJTiYoPPPx5GaSvnGjVhDQrDFxtbc4hUXR8DYRkxmaBrlqD0F013mna0gS7OhlNKthFZCDy9pjpX16XsBCLKHU+TII9w3HKvFWn/klOUoryCc2Qfx6tgR/ztORTmdWIODwO2meNHP+J5yiv6/NyNH7SmIyAIRCal1Hioi81tUqj+Aj48PeXl5+mXVTPj4+ODj41PvNaUUeXl5DV4/mUlISKizZ0RziO05KQCM6GC05jv4NbJKKGUZzvBhKIcDW2wsoZdfRtjVV+EVHw+Au6QEn37ty3mOp9eNpswpRCilCqpPlFL5IuKxWwXj4uJITU3VO3FbCR8fH493GtIWjBrV5ovzPJZ9hfsBmNztTBakftvwfEJRBuQl4ew+CdiBvfYQUZcuNcftTSl4et1oilJwi0hnpdR+ABHpguE5zSOx2+0ev2NQozmZSStJBZc/IzoaNsIaVAqpxiS0QwxlUFspWIOCsIaH48rLw6df35YV+CSjKUrhQWC5iCzFWNs6FripRaXSeAwzZswADLMYmqajy61h8isz8SaSIK8g7km8hxExI+qN585KoijZF2e4sfvfHtuxznWvrvFUOp3Y21lP1dPrRlMmmueJyBDgVIwewt1KqdwWl0yj0ZyQlLqzCbd3B+C6ftc1GK94+VoyVodi2fYxluDgIxzmhF19Dc7sbD3J3Mw0qBTMYaICpVShUipXREqBKUAvEXm1PZrS1mg0bcvm1Dxc1oNE+R59yWi1n2V3aSne9eyYD5o08YgwzR+nsZ7CF8BFQKGIDAK+BP4NDAT+B/yp4Vs1Go3GoNxRxdSZf8evYjxb0orxjndzevejzwM4cw9i8bLg1asfPtqMSqvRmFLwVUqlm8dXA+8ppZ4XEQuwscUl02g0JwQL92wj1bWIAKcXI3r2ZaMDhsf1Oup9jvwS7GEhxH/+WY1pC03L05hSqD1QdwbwDwCllFuP4Z089Gtny/08BV1uh/h1/24ABnev4NRYCxvXQnxQfOM3VRbjLHFjiw1FrA1sbGuneHrdaEwp/CwiXwAZQCjwM4CIxAB6PuEkYdiwYW0tQrtEl9shtmUb7mEPlOwjuiiCYO9gQn2OYr20MBVHmRWfE9B7mqfXjcaUwt3AZUAMMEYp5TDDozGWqWpOAqqdAdnt9jaWpH2hy81AKcX+olQIhP3F+wnyDjp6LwFQOftwVViwdezU8kK2Mp5eNxpUCsqwE/F5PeEbWlQijUdR7ZXNU9dUeyq63Ax+zyujghzsgFu52ZKzhQu6N+wsqhrn/p2AYO/Uo8VlbG08vW7o2RuNRtMsuNwuiivLKKl0UljmoLDMwYq9uVjsB4n2NTaeKRTxwfH1J/DNLbD2fQAc+w2jebYuPVtDdE0tmrKjWaPRaBrlnE/v5IBjMUoJZSl/xl1RPeyjCOyVz7hOU/g66Wucbmf9w0cuJ2z+Anb+AH0vxJmeCoA9JqbV8qAxOCalICKhQCel1OYWkkej0bQz3lozlwOOxYRLInms5eyhpQwPNfYhlLkKeTOlkq4hXeka3JU9+XvqVwolmaBcUFkEi5/EYSoFW3R0K+ZEA00znb1ERIJEJAxYD7wtIi+0vGgajcbTqXBU8frmF7E4w5k97X9E+UURGpLL/43pyv+N6cq4vsZy0o4BHekZ0hOLWOgc1PnIhAoOGN8hnWHNOzizsrF427AEBLRibjTQtJ5CsFKqSET+BHyolPqXiOiewknCoEGD2lqEdsnJUm4v/voNTlsG13R9mCAfX3qF9mJX/q6a66klRos/LiCOK3pfQZ+wPnhZvQ4lsPBR6Hk2FJpKYcrrkLoGR8YmbM6cE9KukafXjaYoBZu5N2Eax7AUVUTeA84DspVS/c2wMGAmEA+kANNM/wwC/Bc4BygDpiul1h9DPjQthKdXYE/lZCm3H1PmIK5g7h41FYBeYb1Ymb6SMkcZ7219j935xsa1joEd8bX5Mihq0KGbHRWw/AUozoAIY0JZxQyiojSUytRF2KMbcb7TjvH0utGU1UePAfOBJKXUGhHpBuxpwn0zgEmHhd0PLFJK9QQWmecAk4Ge5ucm4PUmpK9pBcrKyigrK2trMdodJ0O57S/IIV9tISFgLF42o33ZK7QXTuXkjc1v8ObmN1l8YDFRflH42nyPTKAk0/jO2mYMH/mGUbx4OSkXX0LV3r34DDilFXPTenh63WiK6ewvMYzhVZ8nAxc34b5fRCT+sOALgfHm8QfAEuA+M/xDc2/EbyISIiIxSqmMJuRB04J88cUXgOeuqfZUToZye2XVLERcXDdgak1YQqjhZvLj7R/TMaAjT499Gm+rd/0JFJl/75xd4BcOIZ2o3L0HROi+4CfsHTvWf187x9PrRlMmmp81J5rtIrJIRHJE5OrjfF6HWi/6TKC6f9gROFArXqoZVp88N4nIWhFZq11uajRtxy9pC7E6ozm319CasM5BnfG2euNwO7iox0UMihpEn/AGLJwWm68CVyUcWAXBnXCkp2OLisIrLu6EnE9oDzRl+OhspVQRxvxACtADuPePPtjsFRyzW0+l1FtKqUSlVGJkZOQfFUOj0RwHmw4UUKL20yd0IJZaFkxtFhs9QnpgEQtTekxpPJHizEPHjrIapVDb7aam9WnSRLP5fS7wpVKq8A9o8KzqYSFz8jrbDE8Dahs5iTPDNBqNB/K/XzZhsZVxRvcjLX5e1usy0kvT6eB/lIni4gyw2ClOtZGzKYD48dE40tfhO3BgC0mtaQpNUQpzRGQnUA7cKiKRQMVxPm82cB3wtPn9Xa3w20Xkc2AEUKjnEzQaz+DBBR9QUBhGsDUeAKXg5+Rt+HSG3uFH2ia6qOdFRyay7gPoNRkCalk9Lc6EoFiKMt1UFrqoLLDiyMwkaPLkFsqJpik0ZaL5fhF5FuNF7RKRMoyJ4UYRkc8wJpUjRCQV+BeGMvhCRG4AfsdY5grwI8Zy1CSMJanXH0deNC1AYmJiW4vQLjlRyq3C4eK7A69BRVd8Dt5YEx4RWkgJNGzHqDYFB+D7OyH5Irh0xqHw4gwIjKE8OxNwUbIjC5xO7B1P7OEjT68bR1UKIuIH/BnojLFcNBboBcxp7D6l1BUNXDqznrgKuO1osmhan/79+7e1CO2SE6XctqQdRKzl+AT9zm+3nI7VYuxQfn7tJj7d4UWsfxNe4CWGr2W2fQOj7oCO5sR0cSYOn544CvYb0dZsAzjh5xQ8vW40ZaL5fQynOqPM8zTgiRaTSONRFBYWUlhY2NZitDtOlHJblWKM4la4StmZv7MmPKUwhS7BXWqURKOUmFOHYjV2MFdTnEl5jtEutfj5ULHdSP9EVwqeXjeaohS6K6WeBRwASqky6rrq1JzAfPPNN3zzzTdtLUa740Qpt/Vp6TXHazLW1BzvK9rXJGc5wKGeQuL1sG8p5P8OlcVQVUxZajni50fg5HNqop/oSsHT60ZTlEKViPhiLh8Vke5AZYtKpdFoPILtGVk1x6szVwPgcDlILU6la3DXpiVS3VMYcYvxvWM2FBvpliXn4TvwFHz6GlZVrSEhWPz8mkd4zXHRFKXwL2Ae0ElEPsEwT/H3FpVKo9G0OVlFFeSUFQDQPbg767PXU+oo5UDxAVzKdWw9Bd9Qw75RzCDY9i0UZ+CqEir3Z+E3ZCg+vXsDJ34voT3QlNVHC0RkPXAqxrDRXUqp3BaXTKPRNAulVRXc8v2LdLSciV2a3grPLKpArOUAXJJwCc+ueZaLZ1+Mv90fOGTS4giKMqA8Hzr0NQXIhgBzz0LfC2HRo5C6hvI8L3Ar/IYOwTvBSOtEX3nUHmiqkx0fIN+M31dEUEr90nJiaTSa5mLm5l/YWPIpGw8WYi85YvFfo3SMgIPAhC4T6Bvel8d/exyF4onRT9ArrFf9N/3wN9j1Awy7ESY+ZQwfVe9PqFYKi5+iLMcHrFZ8ThmINcCfgDPOwH/0mD+WWc0fpilLUp8BLgO2AW4zWAFaKZwEjBw5sq1FaJd4UrntyvsdgPguO5l90b+PyabQjK1pPL8OgryCiO4QzTcXNmGCNGMj+EfBmrchdrAxfBQ3zLgW3h331BlYkhdQvmoTPr1jsAYYPY9O/3vtWLPWLvGkulEfTekpTAF6KaX05PJJSK9eDbQGNY3iSeW2v8iwNZlSvJdd+bvoHda7yfcWVRVhFWv9pq/rozwfitJQ4/9J3puvEpK0BltJtqEkAEdWNvtuepbgqVMpT1tJyLQhx5yf9o4n1Y36aMpEczJgb2lBNJ5Jbm4uubl6CulY8aRyy67IQNyB2Cw2vt/7faNxyxxlJOUn1ZwXVRUR5BXU9N5F1nYAipOd5GzwoXjJCsPYnTl8lPu//+EqKODge++hKirwGzK0sdROSDypbtRHU5RCGbBRRN4UkZerPy0tmMYzmDNnDnPmNLp5XVMPnlRuhY5M/Inj9E6n8/Wer0kradjW5Htb3+OKH67A4XIAhlII9Ao86jNcBQVUJu9DZWwBIH+x4bHXkW6aMAvoQFVKCgVffUXw1Kl4dekCgO+QwX8ka+0ST6ob9dGU4aPZ5qc2x2zyWqPRtA2V5NDBewT3JN7D1NlTeWDZA7w38b16dyNvzNlIhauC7PJsOgZ0rOkpHI3UO++ibPVqLD42wvpEUrbB8KZbVWK8YpR/JJn//jfi5UXUX/+Cq7CQ0lWrsEdFNZaspg1oSk8hRCn1Qe0PENrSgmk0mj9OVkkhWEuJ9Y8jNiCWfwz/B+uz1/NN0pETxkoptucawz8ZJUYLv7iq+Kg9BWd+PmVr1xA4Zhg+kVZyN9jBYsG7S0ccpYbiKVy+ndKlvxD1l79gi4jAu3t3wq68splzq2kOmqIUrqsnbHozy6HRaFqADel7AegW0hmAC7pfQP/w/ryz5R2cbmeduPuL91PsKAYgo9RQCkWVRQR5N95TKP3hC3ArwgMW0nlsBhETexF5x+34Jg7DUWrF5RCyXv0Qv2HDCL36qubOoqaZaVApiMgVIvI90FVEZtf6LMZYuqzRaDycbdn7AOgTaYzhiwg3nXITaSVpzN03t27c3G01x5mlhle0o/YUKooomfkKVl+FT9cOiLuCyGunEHHrrdg7d8VVaaUs2wd3aSnhN92EWJrSDtW0JY3NKfwKZAARwPO1wouBzS0plMZzOO2009pahHaJp5Tb3nzDLPWQ2O41YeM7jSchNIEZ22Zwfvfza8K35W3D2+qNt9WbjNIMlFKNzilkPPIIVRuXUZHiIHD8GOTaf8LCf0GPswCwxxlu1osyQgDwHeDZJqNbC0+pGw3RoFJQSv2O4QjHs3daaFqUbt26tbUI7RJPKbe0klRw+RAXFF4TJiJM7jqZ/67/L/kV+VQ4K9iYs5GNORvpHdabKlcVGaUZVLoqcbgdDfYUSpYsxZmZCVgIOO8yCO1Sx4mOV1ycES/NB3uXDlhDQlowp+0HT6kbDdGgUhCR5UqpMSJSTN3VRoLhF+foSxI07Z7MTGMYITo6uo0laV94SrnlVmZgVxFYDhu2GRxlLAXdlLOJH5J/YF7KPACu7H0lmaWZ7C/eT1FVEQBBygJJC40be0wAQLlcOLOzCelRit+FNxM44UjzGXZTKbjLKwkYcEqL5K894il1oyEaG+C7CkApFaiUCqr1CdQK4eRh3rx5zJs3r63FaHd4Qrm53W6K3ClEeHWB5CXgdtdc6xfeD5vYWJu5lhVpKxgZM5LLel3G1J5TiQmIIb0knaJKUyksfQ4+vtj45BkT187cXHC78YkNJviGfyDWI5e3WsPCEF9jJ7QeOjqEJ9SNxmhMKdSsWRORr1tBFo1G04xsztoP1mKG+YXBhxfC1kN/Yx+bD33C+zBrzyyKHcVc1usyHjr1IXqF9SLGP4YyZxnppYaDnaCyPDjlcuPGjI0AODOM1Um27v2hgcljEamxeuqjewrthsaUQu197Z49CKbRaI5gQZLhKe00/wAjYM98cDlh5WtQdpBBUYModhRjt9g5NfbUmvti/GMA2J2/G4AglxtG3wkWO2SYO5X3G6ua7J0PTWDXh1fHOMMSap+m21vStC2NrT5SDRxrNJp2wNrMzShlYbS32fZLWmT0FuY/ACVZDOo5ho/4iGHRw2p8JMAhpbDr4C4AAr2DIKovRPWGTMOMhSNlBwD2bv0alSHogvPx6t4di28TDepp2pzGlMJAESnC6DH4msegJ5o1mnbB7yU78VZxBJi7kyk/CD89aBxv+JghI27Cy+LFWV3OqnNfTIChFFZlrAIgKDYRRCB6oNHbUArn/mTE6sbSqW+jMgSfey7B557bvBnTtCiNLUk9cuZIc9Jx5pnH5pRFY9DW5eZ0uSglhXjf0VB4AELjoWA/lOZAn/Nhx/dE7FvBvIvnEeEbUevGSsLKCgnyCqLCWc61hUWEDB1tXIseABs/huJMHBnp2P3cSLgeWT5W2rpuHI2mel7TnKR06tSprUVol7R1uc3dtQUsFQyIGIDauYKyinj8OkYhObtgyuuQvQPWvkfkwMugJAdydkKX0TDzaiwpy/n69pX4p6wk8JtboMto3FVVVOR64weQuQVnzkFsgRbwProFVU1d2rpuHA2tFDSNcuCA4aDF0yuyp9EW5bY3L5uHf36bGDWJH/bNxR4NF/QeQcWP/2L/3BJiH76bwKm9yHjgUSITz8Br97tQVQY/PwbrP4SofpBtmLqILsyE9I1g84WYUzj41jvk/Pdlup1jxTtzE478Evw7B7Ra3k4kPP0/pQ2RaBpl0aJFLFq0qK3FaHe0drm53YqbvnuRLWWfs3j/r3SJycPH6suw0Cgq8wzDd8Vrd1K0vYCiH36gaI8DlAuytsKB1RAYY/QWep9nJJi6FvavhLhEFBbyv/wSgPKKTqg9P+MscWGLimhIHE0jePp/SvcUNJp2hNPlZsW+VBYmr6KrX2JN+O6sQjJdK7BYYfoZilUZ2cRY+2ItSqOq2Pibl/6yDFdBAQBlKfnQGWNTW84uOP1BSLwefMPgvwMhebGx0mjsPZT++itO01lOubsH/jt/BhWNPbZjK+de0xpopaDReCi3ff9fFqcupPLALYg9F6t/ElX5p+IV+T1e4Sso2fMAymksArT67sMvPh+b2Pgt4zd2HdzFFb2vgIIDNY5u3KWllK38DWw2yrftRPXpgKybASiIGwr+Zss/LhG2zTKOO59KwX+/wBoWhnfPnpSnH8TR3XimvZOeZD4R0cNHGo0H4nS5WJb9FVa/FK4a7c+Afuvw6vAtl4xyENbBcITz5v/FsumfZ7Ppn2dz2enZ+Np8uTjhYrbkbqHKXUX/iP5QeICqYht+iYMRb28AQq+4AndhEVVefaEoDUeplaJdxSi3m6KffiJ7lROlALHgjhhAyZKlBJ9/Hn6JiVQm7aVUGT0Ue4LepXwiopWCRuOBfLBhIcpWAMDAnnmUWIzNYluq3qDEmQ9AWlkywX52LLYKFh2Yz9ldzmZsx7E1afSP6I/K309ViQ3vPv0JGD8e74QEQq8wTFaUlxiWU3OSOpL21/tJPv8C0u68i7w5a3FWWCB6AOU79qIcDvxHjcJ30EBwu8n9aSf+/bvgfep5rVgimtZCDx9pGmXSpEltLUK75I+W28wd34DbhxBfX2YlzSKzNJMY/xjSStII9ArE2+pdY4Zi1p5ZlDnLuKrPVcQGxCIIwd7BdAzoiDNtH8opeMV3Ieovd6OcTiyBgViDgynLdBMSDhUHvbHHRePKzcU3cSjla9dRWeyPvcsYSletAqsV36GJ4HQYwrndRP7ruQZtHmkax9P/U22iFEQkBcNZjwtwKqUSRSQMmAnEAynANKVUflvIpzmEp5r39XSOt9z+Ovd1FmZ8jttSQFf7aHqGuVmQsQKAJ8c8yc0LbmZS/CQySzPZnb8bp9vJJzs+IbFDIn3C+wAwIHIAUb5RiHLj2LEesOPVJR6Ln1/Nc3wHD6Z8TzLuyEAqc8oIv/kaIu+4A3dxMbtHnEpFz1sIGH8HZR/cik//flgDDDMYPgMG4NW5s7Z6+gfw9P9UW6r605VSg5RS1Uso7gcWKaV6AovMc00bk5ycTHJycluL0e44nnJLyS1l/u/fYxFFJ/tpPONnIXH7fACi/aNJ7JDIzPNm8tehfyUhNIHkwmR+3PcjGaUZXNv3WsPYXe4e/nfm/3hizBOQupaqnFIAvOK71HmW/8hTqUrZT/HAV8Ct8O3fH7FYsAYHY4uNofJAPm6XjfItW/AfPqLmvvhPPib2maf/YOmc3Hj6f8qTho8uBMabxx8AS4D72koYjcEvv/wCeL63KE/jaOWWlJvF7qxCAr1Ca8Le+GUH4pPGtF7X8sDIv8GHF2IrLYbQAEZEj0BE6BnaE4CE0AScbifPrnmW7sHdGRc+AD6eCvuWEnzrr9ChH+z6kapSO2K3Y4+JqfP8gPHjyfr30+S+9S4APv0OGbbz6dWbil27KFu/AZxO/EYcUgri5dU8BXQS4+n/qbZSCgr4SUQU8KZS6i2gg1LKtNxFJtChvhtF5CbgJoDOnTu3hqwaTbNz9ey7KXJmU5Z8N9UddqtfMn5d3IyOMzvPuUn0cLq5urCY8zrU9YqbEJoAQGFlIQ+NeAjLZ5cbO5ABUlaYSmEuVa4o7J06HOEEx6tLF7y6dqVq716s4eHYOhz6u3n37kXJ0qXkf/45Fj8//IYMboki0HgobTV8NEYpNQSYDNwmInU8WSulFA2Y61ZKvaWUSlRKJUZGRraCqBpN81JQVkWxKw2rdzYPXKL46paRfHXLSK437aQNihoElSVQlIplxC3cV1hKv+Rf66TRJbgLdoud7sHdOSuwG6SugQmPGDuTU1dD3l5Uzi7KswWf3r3qlSNg/HgAfPr1ReSQ+xSf3n3A7aZk0SJCr766zlyE5sSnTZSCUirN/M7G8PA2HMgSkRgA8zu7LWTTaFqapXuyEHshAMuyvyQxPozE+DAyKnbSLbgbwd7BkJdkRO48AjomGiYnamG32Hl01KM8OfZJrDt/BKDSfzAHloXi2rsKts3CUWLFmV+CX605gdocUgp1fSJUKxHx8yPs+unNlGtNe6HVlYKI+ItIYPUxcDawFZgNXGdGuw74rrVl02hagwU7dyPipndobzbmbGTR/kW4lZuN2RuNXgIcUgrhPaHLKGNoqLKkTjrndz+ffuH9YOcciBlI3qezKdlZQHlSFqx9n1KHsRrJb/jweuXwGzqEsBv+j5CLLqoTbu/UCXtsLOF/ugFbaGi992pOXNpiTqED8I3ZXbUBnyql5onIGuALEbkB+B2Y1gayaQ7jvPP0BqXjoaFyU0rx24EkiIQ/D/ozb21+i38s+wejYkdRVFVEYofq+YTdFKf7Uvb2LDpcPhKWPWfYI/rpYRh+I4y8zYhXlAGpa3AM+SuFrxo+mCsL7QQUpVFW1AtbpBWvrvH1yiI2Gx3uvffIcIuF7gt+0vsQWghP/0+1ulJQSiUDA+sJzwM82/vESUhEhLaEeTwcXm5ZRRXc9P2T5BV5U+Bw4wt0C+nGK2e+wtU/Xs3SA0v588A/c07Xc4wbcveQuzOMil8+IfTKS/ESC/zwNyjJMnwsj7gFLFbD5DVQsM0FTieWgAAqCitQYqN0Tzb+I0fVmS9oKodPTGuaD0//T3nSklSNB7Jrl+Gnt1ev+icrT3Ze+nU2H277AO/y0dgrByLmiGxZ9n4A/KKMFXLZ5WlYO3+Hr28H+gaOYp/L2HvgbfXm03M/pcxRRlxgHOyaC7//StWerVRkG2stSn5bR1jMQEjfAEEdoSgN9iyAqD6w/AUKq0aTN+s7As44A+WoonLnaqqiRuLK24Df8GFtUzCaBvH0/5RWCppGWbnSmOD01Arc1nyx41scXrtxeO2mm/USutouQCnFjuQUADoO9seClSDWk+JQVJBJ19gCSnIj8bYaBurCfMII8wkDpaj6+iEc+/dTftALCMIaHk7J0qWEXTwWMjbBlTPho6mw7HkQoXCfD+kr9uE3fDgxTzzOwffeI+/XlRSrMcAGAkaPbrOy0dSPp/+ntFLQaI6TAwfLKHQn0d33VKJCFKnFK3l2yj+Y9v00OsaGMq7TOD6yv05OWQ5Wi5Xuwd3ZW7iX5WnLa0xS1CFzC5nzD1KaFYFYFb694/EZNpaCL77A+ch3VFb2xLkmBf8el2Lb9BpOdyBZm6LwHdybzu++g9jteCckgNPJwQ8/wmfgKdg7ap8HmmNDzyRpNMfJ5+u3Y/HK58yuw7mw+4WklaTx1Kqn2Fu4l70FeylzlJFdlk3X4K742fx4cuyTWMRClbuKWP/YI9JzrvyY0mxvvBN6oNwWgq/6PwLGjUNVVpI06UL23/tv0u/9O+k/ZMMNC8h2XourvIroRx5B7HYAvBOM1qeroICgSZNbtTw0Jwa6p6A56flm20p2pJcT5tUFt1IoZbi3dCvM81rHQL7jAAcd+9j4ezmEwWldEkkITcDH6sPXe4wVQFllWWSXZYMX3JN4D2M6jkFESAhNYOfBncQGmErB7YLd8yEviaIfvgclxD73PLbISKwhISiHA5++fbF3jCVk2jTK1q8n7/U3yHy/M4WzviH8xj/h0yuhJi/eXePBZgOnk6BJE1u/MDXtHq0UNCc1yXkHeXjV3bgdQZTtu7veOCJgtRciWBBXMLbomViD1oFPAhas9M5KwbeikvFxpzHv958YGDmQJY4lJBcmQwj0COlhrACqLGFgeP9DSiFpkbGiKH8fAMW7w/Hu1BmfhEMvefHyouusr2vO/YYPp+i72eR/+hl+iYlE3nlnXVm9vPDp1Qvx9TnC3pFG0xS0UtA0ykWHbWw60bj7h7cQaxlWaxnf/7UbPUMTsIhgEbCIUFhZwC2LbmF73nbiAuKYe/FcLpszg+15gN9urtlg58AL99H59Dz+dPY1RPS5irHbF7B+aBC5Qbn4iz/R/tHgcsC7Z3GKlDPTD2K3/wCbvsXp24Os9HMo3bYfV95BIu5ofHuOxdub6H/9k7x336PjC8/XDBvVJu6Vl+sN13gGnv6f0kpB0yjBwcFtLUKz4XC5uWfOVyzKeZWy/TeiHCH4dZtLpH88hc5UFhyYy4CouhPAXyd9zfa87YyOHc2K9BVkl2WTXJBMtH80maWZjFpXhqvSyoHlMXQN/5m//98HJN31DpcO8GbW6AOcEnGK0UvY8BFkb2eCzZv0Sm+GJX9HvpxL9ldJqIptBJ1zDrYOHQi98oqj5iNg3DgCxo1r8Lo99sj5Co3n4On/Ka0UNA3icis2bNoMQL9+re9Uxel28/5vG/gw6T8EFF2NVf2xP1NppYt8n5/xCs8noe8CouwJbCg5yP3D7uGH3+fxQ/IP3DX4LqwWY+OWy+3iy11fMix6GDeeciMr0lfwU8pPVLgquDmgN+klgYTm7CB48niKfl5B3qoCwsMfw5lvp8OGciq6WOluz4I178DSZ6HTqfhNfoablzxD+k4vipasxm/YMKIffRTvbl2bo8g07YCtW7cC0L+/Zzoq0kpBUy8Hy8o487MpZPxkwVnSh4ABE9pEDnvIb/jE7KRT5C5irX9MBgF+9yoktdxGWuUG0io3cF638zh78XNgdbDEVsyXu7/k8t6GD+OfD/xMemk6f0v8G332/YYA3yV9C0Cv9Z9zxlY7WQQTfuffcRU/Rtmm5fitXgmEcjC9gqokYcjB3ymY9xD+0VXYp30EsYMoDruOoiV/IfzWW4i8887j2nGsab+sXbsW0EpB0864Y/YMnLYMQkJsjOh6Bv6jkwiyRRPvl1gn3uaiOaRVbCXO5xT6Bp6FVZp3LHtj2S+syIa42AO8cuaQ40pje952vtr9FfcPv58xn+/l4p4Xk1qSSrRfNA93nEjpR+8y2qoYO34wz6x+hu152/k1/VeyyrKI9o/m9E7jKXnqL/yl0MUL5+xi4F43QXsGUJCai1encLy7dsVv1GmULP+NogM+AFjcitiDQsIGPzIqbGC10OGUJPyGBpH5+BP49O9P5G23aYWg8Ti0UvgDKHP5ooJDSxmVYk9WCfcvfoaiiipCqiYheNeYP2gL3JSjcGIl8LDwCpxSgJeq6zPWrdzstX+L1QdKXbkMjLfxUcEM4oPieXryZXXiTvzye7LL80gu+42LTunDhC6Nt+aTC5P5Ne1XrupzVZNeiJfM3gvA6szVOFwO7NamKZ2N2RvJLsvm7PizeXvz2yzcv5AuQV0od5YzUPx4aMIbRl6/vJn0lWEgVp4MXM/1g/syJ3kOp8WdxnUdruOMzmdgy9hC7toqTi2zETpWcfmvbspSDX9Q4bdcCIDfCMMSaUmaL94JPWDNWkbuAqGKmCefpHjhQrKeeAIAS3AwMU8+idj030/jeehaWYtvNm/jkdV3486ehruiEwo3KnA14MBVMAZL6GKsIb9SlXU+juKGu34W70z8u30PXlBk/xk3TgLoQVf3rdg5vnHxLPmJQNUHPzod030KFzstL+Gmgr7uJ2qUk8LNbsurlJFCX/cTeGM4LEqTWZTL71glg/O6nc9Hyz5kfsp8nH2cJBUkkVyYTLdgw43gwYqDpJdlcXtxBa8H+bHj4I6jKoX3t77Pt0nfYrPYaoZplqctp3NgZzoHGXaCNmZv5NGVj/LS6S+RVJBEj+DuJBXuZVPOJhKj6/ZUFiXNZuW+n4jzCuGixLsI8A3jjp/vYFnaMgBes73GktQlALyx6Q16pil6vvcarlf7Y+05gpL5P+J2BIBNKNwYzye+m1FRCQSUKMiaBxn7KNudibPM+KucuVHRLQ3Cb7mZgLFj8enbFwCf3r2xBAXhLioi8KyJWJP3EZGXh71PLMEXTSF4yoXkvf02yuUi7JprsAYFHdPvqNG0FielUvhh51peXf8ug31vxCqGz9kKh4vv9n6Pd0wmoZ3nMDboflaVvEqWYzOCcGmfc5ifv5pydxE+cR8z1P88hgdNx2qxIAIWARFBBJYXLKLTEmFalY2feh7E1n0MXxbvJtvrGT479zMi/SIpqSohwCugSfIeKDrAOd/M5MLuF/LEmIubdM9vGb+RVZpFekk6Gzb9DsCNZzsYGzcWMF7OG9YlYRUrcV0X89y458guy+bML38g2jeKU1Q0D+Zk85lY2Ja7jW7e3civzGfh7wu56ZSbANia+ivXLXAxej/sOL2KXRFr4SieG9ekGR7E/rPmPwzpMIQOfh24Y9FtDPWL453zPkN5B/LimudIKkjisV8fpVuqk8c/3cWD19h4ZeUT+Nv8mNznckaE9+PjhX/ji7wkYgvdzOxgoShvN6NG/Z1lacu4vt90vts7m3uW3oPT7WRwYDwbilM4b60bleVF3otPEHXVBAr22rBFhRN511/JePBB8kLHEOoqI3vJHHxi/PAP+IHCTcGI3R9XeARTVuYgCoImTcKnd++afInVit+wYZQsWoTvkMF4de5MeV4eQeedh5gmqCNuuaVJv51G05aclErh9/xcUh3LSc/qgKV0MO7AFViKx9GvYypJQL4rmcUlf6XSVcUzi5xkFQrrb3iFMncej49+nB252/l012cM7hzCX4ffV5PuvB2fk15ZQMbOxTz4UxUWt4NpG4OIHTKPcy69iyt//4pPd37KiKih3PrzbTwx/EHO7W2sS69wVjB71xfM3fohf+r/J0b3u7wm3Z/3zSU2T7HHuQDGPIFSih/3/cjP+3/moVMfItSnriOUlMIU7lx0B+WuCgDOKihjh92LL9a/wti4seRX5PPKhlc4M24cPf2ieWP3TK7uczVZZVkAvEgk/XfOB//9jB9gZ42PD+cGJbCFChb8vuCQUtj9A+O3KKyVNm6eCc97b6b0rFIeXf4wZ3Q9m4ldJtYZIkotTiW7KIu/pZTwbvdAXlnxGGdEDia4wMXGyt9Jebk/mcOmsz53E0EuF6uz1nD7OjcWJ1y5uorHo/cS5XSy7OCWmjRfnm0jOrmSwmA3b0/ZQlnAe3grxS2/fkyHTn14umIL/Uoc/HXLGm7oHcHQPW6wCAdXZeFrfZ3SzBDCb7mUkIunolxOMh9/gtJNDsALtjgRW0dQLgJP7Qs9hlD0wUeomEi86zFmFjhhAuVr1+I7cBCXXX0Nmcn7CL7g/D9WWTUnHNOmebarmJNSKdw6/Cx2b+nM9k6/0icsn5/3L+KZC87gy5U7OTO5Cqe/my0hijcyorGvLqUr8NG+rfiEejH6twMMfvcDws/pxit8zCUJl9A9pDsfb3yTl9a8QqUdztuisLiFDn+7jYOffUPKIgfRua8z/tJhfLX7Kzas+pjrfnHw7sFHGVdeScDga3h08V9YtmcZEze4eTb1MW7L3spqXx8u7H4hKzZ+wbPvufhpcDGlfV/jPxV7+Xr/AgAKK/J54NSHmJsyl3k7v8BX7Di8/OiYU849SYX8bvFi6KpAKpxV/HPaNtJ7z2WLlwWH28H/LfyFqP35fDo5mi92fUGQdxC+Vm+6r19IruMMnPahTFFJbPZbzYRNs4kecD7PHdzEL6m/cFrcaWRt24B/JUT/82EynnicrkmVfL38CebuX8Dc/QtYHvklj096m20Ht7MpZxN+ZYVcv9DNiI2+BE1Q/GvYJrJTN/P8uy62xgtvXRTLzqSZXLjVwmVrvXlyooNTdymwWTkl2YdFAx8mgkrmrXyWTJyMipqOSn6HoAvOx7VgPkPXO/gkbAkvz1Bk2iycHracvf38uGC5N97ZofxjrwMvh4XoB+4h89//IXVZCJYAf0IuNnpfodOm4TdkCI6sLPyGDKFi+3aKfvyRksWLCLv9IZTLSdEHHxF+9jn1zocET7mQ4PPORex2Yi6bRsSokXjFx7dizda0B/w83Oe1KKXaWobjJjExUVUv7zoWCl76O+lvzObha2z4Virumu1m6Z2nsPXAVu77zA2ALciCqnKAzRdXSRXvT7Aw9KCTU9YbQwH2yCouvTGQy3tfQWJ0Ig/9eBevvO0kNMSCq6wKe2gXuv64CFdRERkPPkjxgoVU9S/nmvMCuOUHN6dvUaxKEHInlHJN1Kk8vHErd/zgxuaAMh/4z1QL27pYCPMJY+CqXG790U1aGBRdVEjR8hASDrogzMnfJvlS4icIwiOzKskLEF45y8bH/3NgLzJk9U5IoKrwIKX5uWy/xEVK/1NI2biDe792oVzCL+dW8f6gEGKCOhObm8df38ygqsiG+PqyPT8fV4APE86Nx+a3mhsGjGRvaQbPn/Ysvzx9B5cuVvRYupSdN1xBUkU6L10p9Eqz0CfUzXuBVv5T5c/zfkKms4T4CjtPvFKOt90XVV7Oa+cLAWXCdYuMMr/veisj9whTljvBYsGFG6sbOjz8EFmPP0GHB/5BwBlnkvvyyzhzc3AWFODMzqHHgp/Y//AD5CycxyenW7h5rhvfQYOo2LYN5XBgCQzEHhNN5e49WIP96fnrKoo/ew2300bglCuxhoQ0qd4ot5u8d98l+PzzsUdHNxp348aNAAwaNOiY66fmxMZD6kaDqzxOSiupgdfdgz3Mn7997+TuOS4CKkCWbGbwToXby0rUPX/Dd8QZWMOj6fjyK9i7d+Pcbd4M2GQhqIci6qYrcOR4cemBKr5L+oaHlz3IzUud+JcJVWkKV76d4MunA2ANCqLjf/9L6OWX4bXVl2c+gdO3KOydOjFit2J9fgCP5qxj+gI3XvHxdH7vXfyiO/LwTDcf74XyikJG7DQUd8eDsLwghhG7FBEBcYQl+/H6h24e3FfE3N3QZ5eFMestzPlFsBdZiLr3Hjq9/TZdv/qS7jO/xO4WqnZa2fn7Vu6a5carW3e8e3RnzC8+OCsqScrfw1WfZOEotdP5ww/otX4dOddcw++BIWR+l0Lp5kjeSs+ku1cIdyy5m64HoDLKH3uHKIJGj6dHOnTbK/zlExcXrRtOD0so99lLyXSW0KuyigEbK/FyQpcPP8Bv2DBunidc8Jsb+vSAwAAenWlhynInwZdcTPznn2Hz8sYroSehV16Jd+/eZD31b/ZOmEDR/PlU/b6fyu07iLjxT1h8fQmfdC4BFXDVYje2+C50+exTei77hdj//Ieus76m09vvYI+NJfSqaxGrlaCr7yRk+p+brBDAcFMZceONR1UIYPzxq//8Gk1tPL1unJTDR9bQKGJfeA3n9OvBZqO8UzDDd+RhcykYPYzwP/2pTvygCetxvPkm2GxEvjYbS1AIOe9/xbnrK/gsrpSELBcjNkLIRefjc8oQ8j/5lKDzD/lhFYuFDv/6F149e8LTz2CJDCX+i5nsu/RSbpufzeLeXoSWKuIefAT/U0fQ89Mv2DflfLy+z+GFM0sJTQnCf9w4Spcu5dJ5ZTjsQsIX31K5Zw+pf/4zQ+dYcPiWYPEPRrmF0l/LscXEEHbddTXLHu3R0ZQO603fzTtI87VgdyriXv0fzryD/H7llVyyCn7tYSU03ULU327D33T27t29G15dbyR49x4K5syhe58sPsxL4vXQCPoesOM/aQwA4aNOo+yDT7lxnhvl60P5ylU8WtKTZ/rmMTZ8MJOz/cles5yq/t3wHTCAji+9SMVFFxGanUPc7XdTuTeZnBdeIOL224m47c+ICPFffIHF1xcRofM7b1O8eDGuvDyCzjsfe2wMlXv24N2zJwD+Y8aArw8B5RWEXToNEcEaEkJwrd+h+0/zQbuZ1Gga5aRUCgD+p55K9L/+iTU4mNySbHz/+QwAkecdaawq8MwzyHvzTYKnXIhXF8McQeDkSbBgATenOThtoRfWIG8i7/0HttBQQi+//Ig0RISwq64iYMwYsFiwhYYS98ILOKdP5+JfHbj79axZ624LC6PTG2+TcuWVhM8xXmKRt99O9uZVhORXUHBaf6yBgfgNGULnDz7k92uuoTzHRcTtN+AuK+fge+8RdvVVR6yD7zL1KgpXPsSFvykYMRivLl3w6tIF/1GjOGvLauwVLrBZCZ52dV3ZLRYibr2Fwu++Iz/kLvwH9+Kyn5ZRWPk9UWOMJai+Q4fiFggthYg7b8IeE0P2s8/y0BY3sI4yi4Xw7t2JefhxI4/h4XR5+22KFy4kYPx4As44g6DJk/DqdGjJbW1robaICEIvvbSOXD61JnstPj4Ejh9P8YKFBE+5sN7fXO8L0GiOzkn9Lwm9wjA+FlBaypbHnwUg7PQj19n7DBhAzNP/rmOELOquuyhdtpwJXwmqrJwOzz+JLTT0iHsPx6tLl5pj31NOoctbb5H+jweIufeBOpOXPn370n3ePAq++AJXQSE+/fvhSOwHC9bR9fLrD8XrlUDnd98l/9NPCbv2WlAK8faqVzFFn3UuOd7/wqvSRezVh9IImTaN0rt/ZfJGKwFjT8Naj8Eury5dCDzrLPLeeZc8QPz8CDr3XALPPAMAa0AAXn1649ybTOgVl2MLDSVo8iRKfvkFW3i4sY7f379uufbqVefFXlshHA8d7ruP0CuuwBYe/ofS0WhOZk5qpVCNxd8f1xXng1JY6lkZICKETJlSJ8zesSNxr77C79OvJ/Csswg655zjerZfYiI9FvxU7zV7dHQde/mDbn2AvMgv6DDu7DrxfAf0x/ffT9WcR911V73pWXx8CJw8icrVawk6/fSa8MAzz8AaEYErN5egc89tUNbIO27HVVxE0OTJBF9wARYfnzrXY/9+P678gzXK0eLjQ9DZZ9eXVItgj45u0ni/RqNpmJNy9VFz4khPxxYZ2W7s16uqKtxVDqwBdVvtOf/7H/kffUyPRQvrKEaHwwGAvZ3kz1PQ5aZpCA+pGw2uPtJKQQMYyy1VefkRQzwajeaEpEGloIePNIAxmSz1KIQ1a9YAMGzYsNYWqV2jy03TEJ5eN07KfQqaprNt2za2bdvW1mK0O3S5aRrC0+uGVgoajUajqUErBY1Go9HUoJWCRqPRaGrQSkGj0Wg0NbTrJakikgP8fhy3RgC5zSxOc6DlOnY8VTYt17HhqXKB58r2R+TKVUpNqu9Cu1YKx4uIrFVKJR49Zuui5Tp2PFU2Ldex4alygefK1lJy6eEjjUaj0dSglYJGo9FoajhZlcJbbS1AA2i5jh1PlU3LdWx4qlzgubK1iFwn5ZyCRqPRaOrnZO0paDQajaYeTiqlICKTRGSXiCSJyP1tLEsnEVksIttFZJuI3GWGPyIiaSKy0fwcn6OGPyZbiohsMZ+/1gwLE5EFIrLH/D66R6HmlalXrTLZKCJFInJ3W5WXiLwnItkisrVWWL1lJAYvm/Vus4gMaWW5/iMiO81nfyMiIWZ4vIiU1yq7N1pZrgZ/OxH5h1leu0RkYivLNbOWTCkistEMb83yauj90PJ1TCl1UnwAK7AX6AZ4AZuAvm0oTwwwxDwOBHYDfYFHgHvauKxSgIjDwp4F7jeP7weeaePfMhPo0lblBZwGDAG2Hq2MgHOAuRjmik8FVrWyXGcDNvP4mVpyxdeO1wblVe9vZ/4PNgHeQFfzf2ttLbkOu/488M82KK+G3g8tXsdOpp7CcCBJKZWslKoCPgfqd+bbCiilMpRS683jYmAH0LGt5GkCFwIfmMcfAFPaThTOBPYqpY5n42KzoJT6BTh4WHBDZXQh8KEy+A0IEZGY1pJLKfWTUsppnv4GxLXEs49Vrka4EPhcKVWplNoHJGH8f1tVLjH8404DPmuJZzdGI++HFq9jJ5NS6AgcqHWeioe8hEUkHhgMrDKDbje7gO+19jCNiQJ+EpF1InKTGdZBKZVhHmcCHdpArmoup+4fta3Lq5qGysiT6t7/YbQoq+kqIhtEZKmIjG0Deer77TylvMYCWUqpPbXCWr28Dns/tHgdO5mUgkciIgHA18DdSqki4HWgOzAIyMDovrY2Y5RSQ4DJwG0iclrti8ror7bJsjUR8QIuAL40gzyhvI6gLcuoIUTkQcAJfGIGZQCdlVKDgb8Cn4pIUCuK5JG/XS2uoG7jo9XLq573Qw0tVcdOJqWQBnSqdR5nhrUZImLH+ME/UUrNAlBKZSmlXEopN/A2LdRtbgylVJr5nQ18Y8qQVd0dNb+zW1suk8nAeqVUliljm5dXLRoqozaveyIyHTgPuMp8mWAOz+SZx+swxu4TWkumRn47TygvGzAVmFkd1trlVd/7gVaoYyeTUlgD9BSRrmZr83JgdlsJY45XvgvsUEq9UCu89jjgRcDWw+9tYbn8RSSw+hhjknIrRlldZ0a7DviuNeWqRZ3WW1uX12E0VEazgWvNFSKnAoW1hgBaHBGZBPwduEApVVYrPFJErOZxN6AnkNyKcjX0280GLhcRbxHpasq1urXkMpkA7FRKpVYHtGZ5NfR+oDXqWGvMpHvKB2OGfjeGhn+wjWUZg9H12wxsND/nAB8BW8zw2UBMK8vVDWPlxyZgW3U5AeHAImAPsBAIa4My8wfygOBaYW1SXhiKKQNwYIzf3tBQGWGsCHnNrHdbgMRWlisJY7y5up69Yca92PyNNwLrgfNbWa4GfzvgQbO8dgGTW1MuM3wGcMthcVuzvBp6P7R4HdM7mjUajUZTw8k0fKTRaDSao6CVgkaj0Whq0EpBo9FoNDVopaDRaDSaGrRS0Gg0Gk0NWiloWh0RUSLyfK3ze0TkkWZKe4aIXNIcaR3lOZeKyA4RWVzPtZ4iMkdE9pqmQhYfviu8NRGRKSLSt9b5YyIyoa3k0Xg2Wilo2oJKYKqIRLS1ILUxd7E2lRuAG5VSpx+Whg/wA/CWUqq7UmoocAfG/o8Wo3pTVQNMwbCwCYBS6p9KqYUtKY+m/aKVgqYtcGK4EvzL4RcOb+mLSIn5Pd40QvadiCSLyNMicpWIrBbD90P3WslMEJG1IrJbRM4z77eK4VdgjWmA7eZa6S4TkdnA9nrkucJMf6uIPGOG/RNjc9G7IvKfw265CliplKrZLa+U2qqUmmHe628af1ttGla70AyfLiKzRGSeGLbyn60lw9kislJE1ovIl6Y9nGq/F8+IyHrgUhG50czfJhH5WkT8RGQUhq2o/4jhA6B77TIWkTNNObaYcnnXSvtR85lbRKS3GT5ODvkT2FC9+11z4qCVgqateA24SkSCj+GegcAtQB/gGiBBKTUceAejNV5NPIYdnXOBN8zW+w0YW/+HAcOAG00TCmDY079LKVXHjo2IxGL4HzgDw2jbMBGZopR6DFiLYUfo3sNk7Iex27UhHgR+NuU+HeNl7W9eGwRcBgwALhPD0UoE8BAwQRlGCtdiGGOrJk8pNUQp9TkwSyk1TCk1EMPU8g1KqV8xdgvfq5QapJTaWyt/Phg7dy9TSg0AbMCttdLONZ/5OnCPGXYPcJtSahCGFdHyRvKqaYdopaBpE5Rh8fFD4M5juG2NMuzMV2Js5//JDN+CoQiq+UIp5VaGyeNkoDeGDadrxfCitQrDXEBPM/5qZdjtP5xhwBKlVI4y/BF8guGUpcmI4elsq4hUGzQ7G7jflGMJ4AN0Nq8tUkoVKqUqMHotXTAcpvQFVpj3XGeGVzOz1nF/s9ezBaPH0u8o4vUC9imldpvnHxyWv2qZ13GofFcAL4jInUCIOuSnQXOCcCxjqBpNc/MSRqv6/VphTszGiohYMLzkVVNZ69hd69xN3bp8uO0WhWEb5g6l1PzaF0RkPFB6PMI3wDZqvViVUheJSCLwXPUjgYuVUrsOk2MEdfPnwsiTAAuUUlc08Lzass8ApiilNolhFXX88WcDaslTLQtKqadF5AcMOzwrRGSiUmrnH3yOxoPQPQVNm6GUOgh8gTG0U00KMNQ8vgCwH0fSl4qIxZxn6IZhVG0+cKsY5ogRkYRawzYNsRoYJyIR5kTuFcDSo9zzKTBaRC6oFeZX63g+cIeIiCnH4KOk95uZXg8zvr+INGSuORDIMPN4Va3wYvPa4ewC4qvTxhiSazR/ItJdKbVFKfUMhuXh3keRX9PO0EpB09Y8D9RehfQ2xot4EzCS42vF78d4oc/FsHRZgTHvsB1YL4aT9jc5Sk9ZGaaH7wcWY1iNXaeUatRkuFKqHMNvwS3mhPhKjDmBJ8woj2Mous0iss08byy9HGA68JmIbAZW0vCL+GGMobEVQO3W++fAvebEcM2EvFku1wNfmkNObuBozujvNofDNmNYFp17lPiadoa2kqrRaDSaGnRPQaPRaDQ1aKWg0Wg0mhq0UtBoNBpNDVopaDQajaYGrRQ0Go1GU4NWChqNRqOpQSsFjUaj0dSglYJGo9Foavh/mv4k1X4B13wAAAAASUVORK5CYII=", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAADgCAYAAADsbXoVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABDyElEQVR4nO2dd5hU5fX4P2dmG1tpS+9IEUSQZkUxoqi/iN1YomKMJbHEJJoYNcaS+FUTjZpiotFgQcGKaOwoRCxBQKRIb7qwwO4CW9g2O3N+f9y7w7BsmYGdsrvn8zzz7J33vvd9zz1z9z33beeIqmIYhmEYAJ54C2AYhmEkDmYUDMMwjCBmFAzDMIwgZhQMwzCMIGYUDMMwjCBmFAzDMIwgZhSMqCEiE0UkL95yHCwiMkFEVkep7LtE5PlolG0YB4IZhVaAiFwsIgtFpExE8kXkHRE57iDKUxE5JOT7RBEJuOWXishqEbmieaRvUIZprhxn1kn/s5s+1f0+VUTmN1DGXBGpdOUuFJHXRKR7A3mHi8j7IrJTRHaLyCIROR1AVT9R1SHNfIsRU+d+aj9vhpy/TUQ2uul5IjIzirLcJSI+t67dIvKZiBzdDOXu8+wdYBlzReTHBytLW8WMQgtHRH4BPALcB3QF+gB/B85s5LKGykpq5PRWVc0EsoFfA0+KyLCIBY6MNcBldeS7AFgfQRnXu3IPBtoDf24g35vAB0A3oAtwI1ASuchR53pVzQz5nAEgIpcDlwKT3PsdC8yJsiwz3bo6Ax8DL0e5vkYRB2vTDhJTYAtGRHKAe4DrVPU1Vd2jqj5VfVNVb3HzjBeRz923uXwR+auIpISUoSJynYisBdaKyH/dU1+7b4E/CK1THWYBu4BhIpIqIo+IyFb384iIpDYgbw8ReVVECtw32hubuMU3geNEpIP7/VRgKbAtMk2Bqu4EXgUOq0euzkB/4ElVrXY/n6rqfPf8PsNgIrJJRG4RkaUiskdEnhKRrm4PrVREPqyVWUT6uTq+2tVPvojc3JCcInKU+9a9W0S+FpGJYd7iOOA9VV3v3u82VX2igTquqNPDWCsiL4d8/05ERoVZL6paA0wHeopIrltGjquXfBHZIiK/FxGve+4QEZknIsVuD26mm77fsyciHUTkLfeZ2eUe9wqRda6I/EFEPgXKgeeACcBf3TL+6hqLP4vIDhEpEZFlIrLfc2A4mFFo2RwNpAGvN5LHD/wc523uaOAk4Kd18pwFHAkMU9Xj3bSR7pvoPkMQIuIRkbNx3rqXAbcDRwGjgJHAeOCOukK4b3BvAl8DPV05bhKRyY3IXgm8AVzofr8MeLaR/A3iNvznAl/Vc7oIWAc8LyJniUjXMIo8FzgZpwdyBvAOcBuQi/N/VdfgnQgMAk4Bfi0ik+qRsSfwH+D3QEfgZuDV2oa2Cb4ALnON1djaBrgB5gET3N+yB5CC82wgIgOATBzjGxbuS8ZlOHrc5SZPA2qAQ4AjcO67dkjnXuB9oAPQC/gLQAPPngf4N9AXpxdcAfy1jgiXAlcDWcBU4BP29qiud+s+Hue3ysHpbRaFe39tDTMKLZtOQKH7plYvqrpIVb9Q1RpV3QT8EzihTrb/U9WdqlrRSF09RGQ3UAj8DrhUVVcDlwD3qOoOVS0A7sb5J63LOCBXVe9x38Q3AE+yt8FviGdxGrv2rtyzmshfl8dcub8G8oFf1M2gjgOwE4FNwENAvoj8V0QGNVLuX1R1u6puwWmE/qeqX6lqJY6RPqJO/rvdntwynEbuonrK/CHwtqq+raoBVf0AWAicXvd+Qj73uvfwPHADMBmn0d8hIr+uT3BX96U4hvx44D1gq4gMxdHxJ6oaaOTea7nA1W0FcBVwnqrWuEb1dOAm95534Azb1f7WPpxGvoeqVtb2yBqQtUhVX1XVclUtBf7A/s/vNFVd4T7jvnqK8eEYjKGAqOpKVc0P4/7aJI2NIRuJTxHQWUSSGjIMIjIYeBhnjDkd5zdfVCfbd2HUtVVVe9WT3gPYHPJ9s5tWl77sNSy1eHEa1AZR1fnum/LtwFuqWiEiYYgb5EZV/VdTmVQ1D7geQER6A0/gGKSGJk+3hxxX1PM9s07+UB1vBkbUU2Zf4HwROSMkLRlnvL6WBu9HVacD00UkGaf3N11Elqjqe/VknwdMxHmTnwfsxmlsj3a/h8NLqvpDtxf2KjAGmOveRzKOca3N62GvDn6F01tYICK7gIdU9en6KhCRdByDcipOzwIgS0S8qup3vzf6/KrqRyLyV+BvQF8ReQ24WVUTcc4o7lhPoWXzOVCF0wA0xOPAKmCQqmbjDHHUbVUPxlXuVpxGoJY+blpdvgM2qmr7kE+Wqp5eT966PA/8kgMcOooUVf0OpwFpznHn3iHHjenouTo6ylDV+yOpyJ1XehlnCKihe6g1ChPc43k4RuEEwjcKtfUV4gzf3CXO6q7vcJ7LziH3ka2qw93821T1KlXtAVwD/F0aXnH0S2AIcKT7/NYOMYU+w3Wf3/2eZ1V9TFXHAMNwhpFuieQe2xJmFFowqloM3An8zR0LTxeRZBE5TUQedLNl4ayiKXOHB34SRtHbgQFhivEicIeI5LpvjHfiNOJ1WQCUisivRaSdiHhF5DARGRdGHY/hjN//t4HzIiJpoZ8wZa+9uIOI3O1OgHrc+/gRzjh9c/Fb9/cZDlwB1Ldc9HngDBGZ7OonTZxJ7vp6aHXvYaqI/D8RyXLv4TRgOPC/Bi6ZhzNk1s7tJX2C8zbeifrnXRrFHUp8D/iVOzTzPvCQiGS78gwUkRNcWc8PuaddOI147XBV3WcvC6fntVtEOuIMXTbFPmWIyDgROdLtQe3BmasKZ3isTWJGoYWjqg/hjJPfARTgvKVdz96x95uBi3HGkJ+k/saoLncBz7hj1hc0kff3OOPeS3Emnhe7aXXl9APfxxnH3ogzN/EvnIm/RnHnO+a4Y//1cQxOwxH8SOPLa+tSDfQDPsQxoMtx3nSnRlBGU8zDmcyeA/xJVd+vm8HtoZyJ05ur/S1vYd//09pVNbWf2qHAEve6b3GGgh4EftLQeL2qrgHKcIfv3KGUDcCntcMybvkTIrjHPwJXi0gXnInnFOAbnIb/FaB2j8g44H8iUgbMBn7mznPA/s/eI0A7nOflC+DdMOR4FDjPXa30GM4y6iddOTbjDLv+MYL7alNIw/9nhmEcLCLSD8cIJje2IMAwEgXrKRiGYRhBzCgYhmEYQWz4yDAMwwhiPQXDMAwjiBkFwzAMI0iL3tF86qmn6rvvhrNCzThYpk2bBsDUqVPjKkdLwfQVOaazmNKgW4AW3VMoLCyMtwiGYRitihZtFAzDMIzmxYyCYRiGESRqS1JdT5PP4kQDU+AJVX3U9V8yE8etwCbgAlXdJY47xUdxXO6WA1NVdXFjdYwdO1YXLlwYFfkNwzBaMXGZU6gBfqmqw3CCsFwnTvjGW4E5qjoIxw/MrW7+03CCkAzC8bj4eBRlMwzDMOohaquPXE+J+e5xqYisxIm4dSaOy16AZ3D8r//aTX/WdXr2hYi0F5HuFgwjeuwoLeVvC15jff6nJFXlkeYvoyEv2iVLd9OpsIZ+PbNov8tPu4oANcnCiuFpFHdwgnx12eZjxNJK1CP4PVCS7WHpyHb4k+t/Kcko9TP2ywqSfeH1VjWMMAoBr1DeTlCPoMDuDl52dfRSmukhpyRAZqmflGqlKlUoyfGys6MXjx8y9gTILAvQYaef7FI/aZWKBJSaJKE027m/1KoAKdWK1w8SUDyun82AR9Da1yt1XsFWF5QhwJBOGYjtDw2L1UV7ABjUJZO8Xsns7OSlXXmAHltryCrxN3F126Omb08ufPg/zV5uTJakuk7BjsBx49s1pKHfhjO8BI7BCA2Wkeem7WMURORqnJ4Effr0iZ7QrZSXls3n5RUf001PZMnuu9mdvo12gQA5BAh4PUGTkFalTFgCpRmwug8Mn72LNB+c7vo0rUiBFB+MXFzBK98TPh0pTFgY4NB1UJ4GXj+kV8GQZeVMP1VY3xMOyYOhm5WMCpg7WjjzbaVnAZSm7ytjvW1/PQ1rffmS/JDZWPy4OtR4IKmOE+WAOPdQ44G0akgLieVVkQI1SU6egCuARwkaCMQRddnOMhToVZqEysEFrGgrfFVYBsDAXUmMWFoZTA8AxVmmw7psTYo4VHlYRN0oiEgmTlSmm1S1JDRqlqqqSGTvUeoEI38CnDmF5pS1tfPr957gnfy/oqJsDMykKl24rTTAeaN/QvKI8yHLsc+VK1fy7RU/wr97NwCenBxe85TT8arL6H/JD0np1RNPRga+7TvY+stfcsm8Zdz94Besn3YK6acfyWF/crwS7/nsM5J/eye/mLGV1EGDqFqzBrxeJCmJ45dUAdDrr38ha9J+4YoPCq1xnJGq30/1pk1UrVtHzbbtJPfuRXKPnnizMvGXlFC9YQOVa9bgzcomKTeXpC5dSOnfj+Tu3RGv0ztQVQLFxeD14snIQDzhjbiudNfcH21r7sNmrauz0ZdeSsXSpdTsKMCbnUXa8OF4s7PjK1wbIqpGwQ1q8SowXVVfc5O31w4LuVGadrjpW9g3OlUvN81oBt5Z8Q3v5P+FoyoruNifyd8zkzi16zgu+uFj4E0O5lNVtt37e/B66ffSTMr++wlFTz9NhwsuIHXAANKGDA7mTe7ahU5X/ZjvrrmW4jdmU1NQQLvRe0MTZxxzDANmv8GORx+l/PPP6Xr77bQ/9xz8xcXs+OMfSRt+WLMbBABJSgr+TRsyhLQhQ+rN127EiCaDOYgI3vbtm1dAo1HE6yX9iLohro1YETWj4K4megpYqaoPh5yaDVwO3O/+fSMk/XoRmQEcCRTbfELzUOnzM+PDP6Ht4aZ+5zHslD8w0ePdL58GApS8/Q4VixfT7e67aXf44bQ7/HA6/+RavnjuuXrLTh83DklOpvAf/3C+jx69z3lPRgbdbrtt37T0dHo+/DCGYSQe0ewpHAtcCiwTkSVu2m04xuAlEbkSJwpSbWSvt3GWo67DWZJ6RRRla7Vs3lXAGa/+gOodU9By5w1ZVTmux5d08cOhJ/8e6jEIZZ98Qt5116PV1aQOOoT2554TPCdeL0lJ9T8qnvR02o0dQ/nnX+DJzCR10KDo3FgLoyF9GQ1jOksMorn6aD4Nr4U9qZ78ClwXLXnaCu+sWYQmF9Ch52v83HM2KeIlp2IDv0kKcFqnUYi3/p981/PT8eRk0/mqq8mafEpwCKaWH/7whw3WmXnccZR//gXtRo0KjsW3dRrTl1E/prPEwHY0tzJWbF8JQBm72Fn0MGet+y3pRS+wx+NhwojL6r2mprCQsvnzaX/W2XS87FKSu3atN19DZBx3HADpY0Y3kdMwjETHjEIro3Dn17QLBDg5rQf/6pTL1h/9hzeOvIRkTzJH9ao/Bnvxm2+B30/OWWc2WO68efOYN29evedSBw+m58MP0eHii5vlHloDjenLqB/TWWJgRqGVsbs6j76+Gm457l48niSuX/Iwb377IZcOu5T05PT98gcqKtj90kukjRhB6sCBDZa7ceNGNm7cWO85ESH79NPx5jS1lqft0Ji+jPoxnSUGZhRaETX+AGWyiz41frp3H8OPR/yYtbvWMiBnAD8d9dP98qvfz5ZbbqF60yY6//QncZDYMIxEw6b7WxHrCoopSa6mW00GeLxcPvxySqpKmHLIFFK9qfvlL/znPyn7cA5db7uNrBNPjIPEhmEkGmYUWhGff7uWgEDvdt0ASPWmcvO4m4PnSz/+mOTu3UkbOpSqjRspevwfZJ9+Gh0vuzReIhuGkWCYUWjh5JfuZn1BKelJmczfuByAwZ0H75fPX7aHvJ9eB6q0GzmSml27kLQ0uv7mN2HV065du2aVu7Vj+ooc01liYEahhfOD126iYM9OKr69hi4dF0FXGNj18P3y+QsLQJWM4ycQKNuDNyuLLrfcTFJubnj1/OAHzS16q8b0FTmms8TAjEILp7RmG0npW/n7pYfy3qqXWFzsJ6friP3y1bjxrDtNnUrGMcfEWkzDMFoItvqoheOXMhAlNWsja6rWcWh1NXTa39VErVHwdu58QPV8+OGHfPjhhwcla1vC9BU5prPEwHoKLZhAIEBAnGAuTy39F3m+Eq6tAtI77pe3psAxCuEOF9UlLy/vICRte5i+Isd0lhhEracgIk+LyA4RWR6SNlNElrifTbWO8kSkn4hUhJz7R7Tkak3s2FOMeJyIVMuKlpMWCHBS35NA9nc5VVNYCElJtsHMMIxGiWZPYRrwV+DZ2gRVDc4kichDQHFI/vWqOiqK8rQ6Nu9y3v771wgbk5QTOx5G5pT67WlNYQFJnTqFHSTGMIy2SdRaCFX9L7CzvnNurIULgBejVX9b4LtiJz7R2aXFJIuX88b/st5eAjg9haQDnE8wDKPtEK85hQnAdlVdG5LWX0S+AkqAO1T1k/iI1nLIL3N6CkOyD+PzS14j1ZuK+v0UPfEEmSedRNrgvfsV/AWFBzyfAJBt4RAjwvQVOaazxCBeRuEi9u0l5AN9VLVIRMYAs0RkuKqW1L1QRK4Grgbo06dPTIRNVLbvcTpiHbN6BN1YVK1ZQ8Gjj1H05L/o8fBDZE2cCDg9hdRhhx5wXeecc07TmYwgpq/IMZ0lBjEfYBaRJOAcYGZtmqpWqWqRe7wIWA/svy3XOf+Eqo5V1bG5B/Hm2xrYuccZPsrtsNc41uxw0iQtjfzf/hZwwmzWFBXZ8JFhGE0Sj1nHScAqVQ2uPxORXBHxuscDgEHAhjjI1qIoK88nSZUOHfoG03yuUcg54wz8BYXUFBXh370b/H6SOh+4EX333Xd59913D1bkNoPpK3JMZ4lBNJekvgh8DgwRkTw3JjPAhew/wXw8sNRdovoKcK2q1jtJbeylsrqAjn4/npwewbSaggIA0o8+CoCqtev27lE4iJ7Ctm3b2LZt20FI27YwfUWO6SwxiGaM5osaSJ9aT9qrwKvRkqW1UlGzi/b+AGTtaxS8HTqQNmwYAFVr15IyoD8ASbk2fGQYRuPYjuYWTJWW0jUQgKxuwbSaHQUk5eaSlJuLJyeHqnXr8GZnAQfXUzAMo21gO5laMFVUkBNQaNchmFZT4BgFESF10CFUrV0b9HtkRsEwjKawnkILpsLjI0NT99mwVrNjRzDWcuohh1Dyn7dJ7tUTT3o6noyMA66rU6dOBy1vW8L0FTmms8TAjEILpdJXTYU3QIZnb0OvgYCzc9ldqps6aBCB0lJKZr9Jh0sPLrraGWeccVDXtzVMX5FjOksMbPiohbJ5tzMklJm0dxeof9cuqKkhqUsXAFIPcVxoe7Kz6fzTn8ReSMMwWhxmFFootX6PMlP3drlrl6PW9hTShg5B0tPJ/dmNJHXosH8hEfDmm2/y5ptvHlQZbQnTV+SYzhIDGz5qoWwp2gxA+4zQlUeOoUjq4hgFb04Ogz/7FE9a2kHXV1RUdNBltCVMX5FjOksMrKfQQvlmy2IAhnXd689ob0+hSzCtOQyCYRhtB+sptFA2FK+kU8DPwB7OJrXKNWvwbc0H9vYUDMMwIsWMQgulwP8dh/qq8XQ5lPLFi9l88SXByGqelJR4i2cYRgvFjEILZHfFHnYmlTGo3AOZuZR99IxjEDIzSenfPyp1duvWrelMRhDTV+SYzhIDMwotkA/XLUEFBqQ4/0Rln8wnffRoev3971Dji0qdp556alTKba2YviLHdJYYRNNL6tMiskNEloek3SUiW0Rkifs5PeTcb0RknYisFpHJ0ZKrNfBZ3hIAhuUOx7d9O1WrV5N5/AS8mRl427ePq2yGYbRsorn6aBpQn+n/s6qOcj9vA4jIMByX2sPda/5eG1/B2J8NBYvJ8fs5pM849nziRC3NmHB8VOt87bXXeO2116JaR2vC9BU5prPEIGpGQVX/C4QbE+FMYIYbgW0jsA4YHy3ZWjo7fRsYWu3D03U4ZfM/JalLF1IHD4pqnSUlJZSU7Bcd1WgA01fkmM4Sg3jsU7heRJa6w0u122x7At+F5Mlz0/ZDRK4WkYUisrDAXZff1iiX3fTx+aDLUKrXryftsMOQEKd4hmEYB0qsjcLjwEBgFJAPPBRpAW09RnNBWQlV3ho60Q5Nzca3ZQvJPeu1n4ZhGBETtlEQkfSDrUxVt6uqX1UDwJPsHSLaAvQOydrLTTPq8PW2jQB0TOmKf/duAuXlJPfs0cRVhmEY4dGkURCRY0TkG2CV+32kiPz9QCoTke4hX88GalcmzQYuFJFUEekPDAIWHEgdrZ2VBZsA6JbdF9/WrQAx6Sn06tWLXr16Rb2e1oLpK3JMZ4lBOPsU/gxMxmm4UdWvRaTJpS4i8iIwEegsInnA74CJIjIKUGATcI1b5goReQn4BqgBrlNVf6Q30xb4tnA1AP1yh+Lb4nSmkntEv6cwadKkqNfRmjB9RY7pLDEIa/Oaqn5XZyKzyQZbVS+qJ/mpRvL/AfhDOPK0ZYpK1pGsSu8eIyj+n9NTSLE5BcMwmolw5hS+E5FjABWRZBG5GVgZZbmMBiiu3EqPmhqSOg/Et3UrnowMPDk5Ua935syZzJw5M+r1tBZMX5FjOksMwukpXAs8irNEdAvwPnBdNIUyGqZUd9KvpgY69AuuPIrFctSKioqo19GaMH1FjuksMWjUKLi7ih9V1UtiJI/RBCWecjr7kyG5Hb6tW2Myn2AYRtuh0eEjd7K3r4iYL+YEYFd5GXu8fjp6sgBsj4JhGM1OOMNHG4BPRWQ2sKc2UVUfjppURr18vW0TALmpXfGXlBAoLTWjYBhGsxKOUVjvfjxAVnTFMRpj5bY1QOz3KAD0j1KchtaK6StyTGeJQZNGQVXvBhCRTPd7WbSFMupn9651AHTpNAhfvhN6M7l7bAKTnHDCCTGpp7Vg+ooc01liEM6O5sNE5CtgBbBCRBaJyPDoi2bUpbLM2azWsctgagoLAUhqg/6fDMOIHuHsU3gC+IWq9lXVvsAvcfwWGTGmonoXAJ07D8BfVASAt1OnmNT9/PPP8/zzz8ekrtaA6StyTGeJQThzChmq+nHtF1WdKyIZUZTJaIAKXympEiAjuwulBYV4cnLwpMRmYVhNTU1M6mktmL4ix3SWGIS1+khEfgs8537/Ic6KJCPGVAbKyRSF1GxqCgtJ6tw53iIZhtHKCGf46EdALvAa8CrQ2U1rlAZiNP9RRFa5QXZeF5H2bno/EakIid38jwO6m1ZOVaCCjADg8ZpRMAwjKjRpFFR1l6reqKqjVXWMqt6kqrvCKHsa+8do/gA4TFUPB9YAvwk5tz4kdvO14d5AW6KSKtLVcWlRU1hgRsEwjGanyeEjEfkAOF9Vd7vfO+DEU57c2HWq+l8R6Vcn7f2Qr18A50UqcFumCh856gXAX1BIUufYTDIDDB48OGZ1tQZMX5FjOksMwplT6FxrEMDpOYhIl2ao+0dAqEvE/u7S1xLgDlX9pL6LRORq4GqAPn36NIMYLYdKTw3d/MkEyssJlJfjjWFP4ZhjjolZXa0B01fkmM4Sg3DmFAIiEmx9RaQvTpCcA0ZEbscJpjPdTcoH+qjqEcAvgBdEJLu+a9tyjOYKCZBGCjXuctSkzm3r/g3DiD7h9BRuB+aLyDxAgAm4b+oHgohMBb4PnKSqCqCqVUCVe7xIRNYDg4GFB1pPayMQUMo90M7bjpoCd+NaDHsK06ZNA2Dq1Kkxq7MlY/qKHNNZYhCOm4t3RWQ0cBROD+EmVS08kMpE5FTgV8AJqloekp4L7FRVv4gMwInRbMteQ9hdWUGVR2hHBjWFBQAk5dpEs2EYzUuDw0ci0ldEcgBcI7AHOAW4LBxX2m6M5s+BISKSJyJXAn/Fcar3QZ2lp8cDS0VkCfAKcK2q7jyI+2p15O/eAUBGUtZeFxe2+sgwjGamsZ7CS8DZQLGIjAJeBv4PGAn8HfhxYwVHEqNZVV/F2QNhNEDBTsfvUUZqDv5theDx4O3QIc5SGYbR2mjMKLRT1a3u8Q+Bp1X1IRHxAEuiLpmxD7uKnZ8iu10HagoK8XbsiHi9cZbKMIzWRmNGITTw7/dwN5qpaiAWMYGNfSku2w5AdnpnagrXxXzoaPhwc4wbCaavyDGdJQaNGYWPROQlnOWiHYCPAESkO1AdA9mMEEornHmEjtld8eXNJalbbOIo1DJu3LiY1tfSMX1FjuksMWhsn8JNOP6ONgHHqarPTe+Gs0zViCFlFc68e+c9SVStXUvG0UfHtH6fz4fP52s6owGYvg4E01li0GBPwd1DMKOe9K+iKpFRL+XVuwFIX7yOMiD7tLpupaLL9OnOPkNbQx4epq/IMZ0lBuHsaDYSgMoaJwpq9dz/0e6II0ju3j3OEhmG0Roxo9BCqPSXMbDAT/WatWSfdlq8xTEMo5USkVEQkQ4icni0hDEaplIrGeIuEM441hyHGYYRHZo0CiIyV0SyRaQjsBh4UkQejr5oRijVWknHcmcpsO1kNgwjWoTjEC9HVUtE5MfAs6r6OxFZGm3BDIeVO7byu4+epkqqaV8OeL14srJiLseoUaNiXmdLxvQVOaazxCAco5Dk7k24AFuKGnPu++QpVla9BGmQU+nB26E94on9VJD9w0aG6StyTGeJQTityz3Ae8A6Vf3S9WK6NpzCG4jT3FFEPhCRte7fDm66iMhjIrLOjeE8+kBuqLWxpuQrkgLZpAWUjtVJJMXJ31F5eTnl5eVNZzQA09eBYDpLDMKJ0fyyqh6uqj91v29Q1XPDLH8a+8dpvhWYo6qDgDnud4DTcFxmD8KJ1/B4mHW0WnaVl7FHNjBRO/HGlq0M9PbC27FjXGR56aWXeOmll+JSd0vE9BU5prPEIJyJ5gfdieZkEZkjIgUi8sNwClfV/wJ1XWCfCTzjHj8DnBWS/qw6fAG0d4et2iyvrPgUET9n7lpBjwEnw54avB3NM6phGNEjnOGjU1S1BCda2ibgEOCWg6izq6rmu8fbgK7ucU/gu5B8eW5am+WjTZ8iKowrK4Ljf0XNrl1xGz4yDKNtEI5RqJ2M/n/Ay6pa3FyVu640Ior3LCJXi8hCEVlYUFDQXKIkJGtLlzCoWsjoMhztNpJAcTHeDvEZPjIMo20QjlF4S0RWAWOAOW7ozMqDqHN77bCQ+3eHm74F6B2Sr5ebtg+q+oSqjlXVsbm5rTdw/fbSPVR5NnNsxW4Yfjb+3bsBbPjIMIyoEk6M5ltF5EGg2I2hXI4z/n+gzAYuB+53/74Rkn69iMwAjnTry6+/iNbPzCVfggQYVl0Nh51DTaEzNZMUp4nmsWPHxqXelorpK3JMZ4lBk0ZBRNKBnwJ9cFYF9QCGAG+Fce2LwESgs4jkAb/DMQYvuTGbN+PsfwB4GzgdWAeUA1dEeC+tijkbHGe0Q9oPgo4D8K/5H0Dcho8OO+ywuNTbUjF9RY7pLDEIZ/Pav4FFQK3DnS048ZqbNAoNxGkGOKmevApcF4Y8rZ6Kaj+7d39Bu+wAfQ93Fnr5dzk9hXjFZS4udqaScnJy4lJ/S8P0FTmms8QgHKMwUFV/ICIXAahquVg8zmbnleWf8vjCl+ih51BemUx26lp6+wJ4jrgEgJpduwBIitOcwuuvvw6Yr/twMX1FjuksMQjHKFSLSDvcVUIiMhCoiqpUbZDpy2ezQz6i2v8VU8qP4JWOVRyZOQRSMgDw73SMgrd9+zhKaRgHh8/nIy8vj8rK/deq1MZoXrlyZazFarWkpaXRq1cvkpOTw74mHKPwO+BdoLeITAeOBaYekIRGgwTKVpHlCeDzFPFB+/cp9yRx6JApwfP+nTvx5OQgEfy4hpFo5OXlkZWVRb9+/ag74FBY6MQh72xegJsFVaWoqIi8vDz69+8f9nXhuLn4ADgHxxC8CIxV1bkHKKfRAAH/Ngb6/Dwy8mcUJKcCMLTnkcHzNbt22sY1o8VTWVlJp06d9jMIRvMjInTq1KneXlljhNNTAEgDdrn5h4lIrQsLoxnwB5Q9nkr6BlI5avTV3JXdlZfXvMyg9oP25tm1O26TzIbRnJhBiB0HoutwlqQ+APwAWAEE3GQFzCg0E/nFFZQm+engbw/AmYecyZmH7N0KoqrU7NhBSgRdwObm6KOPjlvdLRHTV+RkZmZGvY6ioiJOOslZ/Lht2za8Xi+1m2AXLFhASkpKROWtWrWKK664gsWLF/OHP/yBm2++udlljjXh9BTOAoaoqk0uR4k12wuo9AidkuvfoV328VyqN2ygw4U/iLFkexkyZEjc6m6JmL4iJy0tLep1dOrUiSVLlgBw1113kZmZeVANeceOHXnssceYNWtW8wiYAITj5mIDYLObUWTtFme1Rfecff3/qd9Pdd4Wtt93HymHDKTDRQ1t+4g+hYWFwYlAo2lMX5Hj8/nw+Xwxr3fOnDkcccQRjBgxgh/96EdUVTnvv/369eNXv/oVI0aMYPz48axbt26/a7t06cK4ceMiWt2T6ITTUygHlojIHEKWoqrqjVGTqo2xvfAbAHrnDgymaSDAxnPOpWr1agD6TPt3XFcevfWWs1fR1pCHh+mrae5+cwXfbC0Jfq81CAfTwA7rkc3vzhgedv7KykqmTp3KnDlzGDx4MJdddhmPP/44N910E+BspFu2bBnPPvssN910U/B3bc2E01OYDdwLfIazs3kRsDCaQrU1Sko2ANC1094hh4olS6havZqOl19Gv1dfIeOoo+IlnmG0Wvx+P/3792fw4MEAXH755fz3v3unSy9ye+cXXXQRn3/+eVxkjDXh9BTaq+qjoQki8rMoydMm2VO5FVIgt8te3y/Fb76JpKXR+YYb8WZmxFE6w4gOdd/oE3GfQujqnbayaiqcnsLl9aRNbWY52iTvrVnC1a8+RVWgiNSAkpXVCwD1+Sh9512yvvc9MwiGEUW8Xi+bNm0Kzhc899xznHDCCcHzM2fODP5tKyvKGuwpuL6OLgb6i8jskFNZ7B9iM2xEZAgwMyRpAHAn0B64CqiNnHObqr59oPUkOit35HHz/J+injIOSUuik3oQj2Ojy+bNw797N9nf/36cpTSM1k1aWhr//ve/Of/886mpqWHcuHFce+21wfO7du3i8MMPJzU1lRdffHG/67dt28bYsWMpKSnB4/HwyCOP8M0335CdnR3L22hWxHFOWs8Jkb5Af+D/gFtDTpUCS1W15qArF/HieF09EsdVdpmq/inc68eOHasLF7a86Q2/P8Axz57PHjYgHkeNR0g6z172P3zbt7Px3PPwZmQw4M3ZSITrpqPFhg3OvMeAAQPiLEnLwPRVPytXruTQQw+t91ztqp/U1NRYitQg/fr1Y+HChQk1nHUgNKDzBsfCGuwpqOpmnHgH0ewznQSsV9XNbWW8DuDjjcsp96zhpPaXUFQ9j6/L8xhSmEbeDTdS+c03BMrL6fvvpxPGIIA1bpFi+oqcRDEGbZ0G5xREZL77t1RESkI+pSJS0tB1EXIhjj+lWq4XkaUi8rSI1OvToTXEaJ63cRkAP8p/k+9/uxSAUd94KP34Y5J79qTXo4+SOmhQY0XEnG3btrFt27Z4i9FiMH1FTrz2KTTEpk2bWnwv4UBobKL5EgBVzVLV7JBPlqoe9ICZiKQAU3AC9gA8DgwERgH5wEP1XdcaYjQv27EaURiyfQWTJ95LelI7cvdkkjZ4MH2ffYbMCcfFW8T9ePfdd3n33XfjLUaLwfQVOcXFxcFAO0b8aMwovF57ICKvRqHu04DFqrodQFW3q6pfVQPAk8D4KNSZEBTu+Ya+Ph9pY6+kw5gf8f55H9CxsJKUfn3jLZphGG2cxoxC6CB/NAZILyJk6EhEuoecOxtYHoU6405xhQ+/fMsgnw+OdqKPZks7fFu2ktKvX3yFMwyjzdPY5jVt4PigEZEM4GTgmpDkB0VklFvXpjrnWg0LNm1jT3IFfWsyoKNja6vz8iAQIKWv9RQMw4gvjfUURtZOLAOHN+dEs6ruUdVOqlocknapqo5Q1cNVdYqq5h9MHYlGua+Ki16+mz/PexkVGNxtdPBc9aZNANZTMIwoU1RUxKhRoxg1ahTdunWjZ8+ewe/V1dUHVObEiRMZMmRIsJwdO3bsl2fatGmICB9++GEwbdasWYgIr7zyygHfTzRobEmqN5aCtHae+2ouy8tfwZPm2OFDB+/dmFa9aTNAQvcUan3QG+Fh+oqcWGz4am7X2bVMnz6dsWPHNppnxIgRzJgxg0mTJgHw4osvMnLkyIOuu7kJx82F0QzM2fgFKAh+UlXpvY9R2IS3fXu87dvHT8Am6N27N7179463GC0G01fkpKSkRBzkpjk4GNfZkTBhwgQWLFiAz+ejrKyMdevWMWrUqOD5RYsWccIJJzBmzBgmT55Mfr4zWPLkk08ybtw4Ro4cybnnnkt5eTngeOC98cYbOeaYYxgwYECz9TjCDcdpHASqSn7x5wylmis8Hflu0PfwJqeh1dX4S0up3rw5oXsJAN999x2ANXRhYvoKg3duhW3Lgl8D6gR29MhBvKt2GwGn3R929uZynX3FFVfg9Xo599xzueOOO+p1niciTJo0iffee4/i4mKmTJnCxo0bAWePxg033MAbb7xBbm4uM2fO5Pbbb+fpp5/mnHPO4aqrrgLgjjvu4KmnnuKGG24AID8/n/nz57Nq1SqmTJnCeeedF4m26sV6CjFg7Y5i9iRvZWR1gNMvm8M1J/wBgG333cfaEyZSsWRJws8nzJkzhzlz5sRbjBaD6Sty/DV+/DX+2NbZDK6zp0+fzrJly/jkk0/45JNPeO655xqs78ILL2TGjBnMmDEjWDbA6tWrWb58OSeffDKjRo3i97//PXl5eQAsX76cCRMmMGLECKZPn86KFSuC15111ll4PB6GDRvG9u3bD1wRIVhPoZkJBAK8tPQLclMHBN8W3ln2IT6PMiJ3LKSkA1CzcyfFr71OUpdcarbmkzp0aDzFNozYU+eNvrgFuM72+/2MGTMGgClTpnDPPffQs6cTMTErK4uLL76YBQsWcNlll9Vb3vjx41m2bBnp6elBQwTOaMLw4cPrNTxTp05l1qxZjBw5kmnTpjF37tzguVDXIA35sYsUMwrNzJ/mv8pzG++hcuu5+ItHMz79IzKyv4QOcOxRPw3m2z1zJlpdTZ8nn0S8XpJ79Iij1IbRNgl1nX3IIYfU6zr71ltvDbrO9nq9wYlqgJqaGnbv3k3nzp3x+Xy89dZbwYnkhrj//vv3i0c9ZMgQCgoK+Pzzzzn66KPx+XysWbOG4cOHU1paSvfu3fH5fEyfPj1ohKKFGYVm5q2NbwLQpfd7nNL5LV5JcSat+msynftOwF9SQsl//sPO56eTceyxpA4c2FhxhmFEkYN1nV1VVcXkyZPx+Xz4/X4mTZoUHP9viNNOO22/tJSUFF555RVuvPFGiouLqamp4aabbmL48OHce++9HHnkkeTm5nLkkUdSWlp68DfeCA26zm4JJJrr7G93F/D/Zk1iXLWyKCVAQITzO43mlOE/pG/n4XTL6MbmS35IxVdfkdyzJz0f+TPtRoyIt9hhMW3aNMBiDoeL6at+GnOdnWiR18x1tnHQ/OWLV0EC/GJ3CV+PPIOSTgO4dszPg+OSO6dPp+Krr+j++3vJOffcFhXe79RTT423CC0K01fk5OTkxFsEAzMKzcqyvJn0Vx/Dvnc3h42dGkyv+Pprip7+N2Xz5pFxzDEtziAAdOvWLd4itChMX5GTnJwcbxH2YZPraaCtYUahmfhudxH5SQVcUJGJZ8y+Ya3z77iDmh0FZJ18Ml1u/mWLMwhgkcQixfQVOYkWea2tEjejICKbcEJ7+oEaVR0rIh1x4jf3w3GKd4Gq7oqXjJHwwqJZBEQY3/U4CGn0K9esoWrtOrr+9g46XnJJHCU8OGrXblsjFx6mr8ipnUA1oxBf4r157URVHaWqtU5DbgXmqOogYA77xoZOaBZ++y45fj8nHD5ln/SSd94Bj4fsyZPjJJlhGEb4xNso1OVM4Bn3+BngrPiJEj41fj+bdT3HVlSS0ntcMF1VKXn7bTKOOpKkFr6CwTCMtkE8jYIC74vIIhG52k3rGuIyexvQNT6iRcabq76kwutjrOZAahaqyvY//pE1Rx2Nb/O3ZJ9+erxFNAzDZfv27Vx88cUMGDCAMWPGcPTRR/P66683fWGErFq1iqOPPprU1FT+9Kc/NXv50SKeE83HqeoWEekCfCAiq0JPqqqKyH6bKFwDcjVAnz59YiNpE7y37gsAjuzqxEjY9eKL7HzqabJOnkTmCSeQc8458RTPMAwXVeWss87i8ssv54UXXgBg8+bNzJ49u9nr6tixI4899hizZs1q9rKjSdyMgqpucf/uEJHXcWIybxeR7qqa74bn3C9ahao+ATwBzua1WMrcEJsKvyJH/PTufxzlixez/b7/I3PiRHo++ijiSbQRugPj+9//ftOZjCCmr8iJxT6Fjz76iJSUlH12Lfft2zfoddTv93Prrbcyd+5cqqqquO6667jmmmuYO3cud911F507d2b58uWMGTOG559/vtGVhF26dKFLly785z//ifp9NSdxMQpuOE6Pqpa6x6cA9wCzgcuB+92/b8RDvkio9PmpqFnPYPVRkz6YvGt+RkrPnvR48IFWYxAgcXaZthRMX03zwIIHWLVzVdMZI2Box6H8evyvGzy/YsUKRo8e3eD5p556ipycHL788kuqqqo49thjOeWUUwD46quvWLFiBT169ODYY4/l008/5bjjjmtW+ROBePUUugKvu1Y2CXhBVd8VkS+Bl0TkSmAzcEGc5AubhZuKKE8ppm91JtsefY7AnnL6Pv003hhEkYolq1evBhzHXUbTmL4iJxBw4ynE8GXquuuuY/78+aSkpPDll1/y/vvvs3Tp0mDAmuLiYtauXUtKSgrjx4+nV69eAIwaNYpNmzaZUWguVHUDsF8cOlUtAlpUHMNFX79DpQcO7TSOsvnz6XTllaQOGhRvsZqdWpe+1siFh+mraeq+0cfC99Hw4cN59dVXg9//9re/UVhYGAylqar85S9/YXKdJeRz587dZ/+E1+ulpqYmanLGk9YzvhEDnln0EWOevIiJf/qQUx9+iu8/fgblG/8FwKHlQ8DvJ/PEiXGV0TCMhvne975HZWUljz/+eDCtNrwlwOTJk3n88cfx+XwArFmzhj179sRcznhibi4i4KVlT1Gdspyrkx8iT1bzXHoGZSl+wEv211uo6tCBdocfHm8xDcNoABFh1qxZ/PznP+fBBx8kNzeXjIwMHnjgAQB+/OMfs2nTJkaPHo2qkpub2+TqoTvvvJOxY8cyZcq+G1e3bdvG2LFjKSkpwePx8Mgjj/DNN9+QneBDy2YUwqTC56MosBy8UJLyHYsyu0BgD0VJXnqn96Bq/mdkTpyIeL3xFtUwjEbo3r07M2bMqPecx+Phvvvu47777tsnfeLEiUycODH4/a9//Wvw+J577qm3rG7dugVDarYkbPgoTF77+lP2eAOIwke9hrEyUM65g86lXVI7jt/dDX9xMZkTT2i6IMMwjATGegph8sWKZwE4pfNo3itaDKpMyf0e5w0+j7THnqcmJYWMCRPiLGX0OPvss+MtQovC9BU57du3j7cIBtZTaJLH//c2Y548h2W+xRxS7ef7I5yA3BNXJ9Pu7OsYmK8EPvyEzBNPxJuZGWdpo0dOTo4FQYkA01fkJCUlkZRk76nxxoxCE7y/6hV8yWspSvZz7qZu9Lz6Qdr5hIlbcqCmhryfXod/505yzmjdO1iXL1/O8uXL4y1Gi8H0FTkVFRVUVFTEW4w2jxmFpqhcwbBqH/MG/Zhj13ehZvO33O49g6F5iicri5qCAjxZWWQcf3y8JY0qCxcuJJHiYSc6pq/I2bNnT5tb/pmImFFohOJyHyXePXSWbDJ7nU3lkq8BOOKLQjxbd9D52mtpN3o07S84H09KSpylNQzDOHjMKDTC1+s3UJgE3TP7UvzGbBCh3ahR7JnnRNVKHz+efi9Mp+stt8RZUsMwwiVWrrOnTZuGiPDhhx8G02bNmoWIBN1oJCJmFBph7cp38fjhqLXp7Joxg/SjjqT9+ecB4ElPJ+3QoXGW0DCMSKh1nX388cezYcMGFi1axIwZM6K2n2DEiBH77Il48cUXGTlyPw8/CYUZhUbYuWMBZ32u9HhhAUmdOtHlppvIdOcO2h1xBGIrJQyjRRGO6+xbbrmFcePGcfjhh/PPf/4TcHwfTZw4kfPOO4+hQ4dyySWXoNq05/4JEyawYMECfD4fZWVlrFu3jlGjRgXPL1q0iBNOOIExY8YwefJk8vOdGGNPPvkk48aNY+TIkZx77rlBVxxTp07lxhtv5JhjjmHAgAFR6XHEvFUTkd7AszieUhV4QlUfFZG7gKuAAjfrbar6dixl21Pl4zcfPEknzxjSPDn4qtYy7Fsl6dAhDHh9VjBf5xuup93IUbEULe5ccEHCO6xNKExfTbPtvvuoWrnXdbbiNLJ7aDhGQVOkHjqUbrfd1uD5WLvOFhEmTZrEe++9R3FxMVOmTGHjxo0A+Hw+brjhBt544w1yc3OZOXMmt99+O08//TTnnHMOV111FQB33HEHTz31VNBw5efnM3/+fFatWsWUKVM477zzItJRU8TjVbcG+KWqLhaRLGCRiHzgnvuzqsY0bt3usl1Me+sORlRn81HRcj7uuInTd6Vwys5UPupYyCH5qWReMGafa3Kvuy6WIiYE6enp8RahRWH6ihw5CGNwoMTCdfaFF17IY489RnFxMQ899FDQhcbq1atZvnw5J598MuD0Urp37w44S5rvuOMOdu/eTVlZ2T5eW8866yw8Hg/Dhg1j+/btzaoPiINRcGMw57vHpSKyEugZazkAXv7iFf65/G62J8PIyiq2ZScBXja3V47OzubdwkrSfBWkh3T32ipLliwB2KfrazSM6atp6r7R1w6RRNOgxsN19vjx41m2bBnp6ekMHjw4mK6qDB8+POhmPZSpU6cya9YsRo4cybRp05g7d27wXKgc4QxhRUpc5xREpB9wBPA/N+l6EVkqIk+LSIcGrrlaRBaKyMKCgoL6soTFvz5+mvtX3kUSfi7MPo5lae3YnuTluJ7H8Y3UsPPiF0jelQVgnk9xGrnahs5oGtNX5JSXl+/jxjoaxMt19v3337+fk70hQ4ZQUFAQNAo+n48VK1YAUFpaSvfu3fH5fEyfPv2g64+EuBkFEckEXgVuUtUS4HFgIDAKpyfxUH3XqeoTqjpWVcfm5uYeUN0vfzaDv296iH4+P/8Y9SeuXN6Tf609jt/rmVw/8joU5YNNH9B5fRHVmakk9+lzQPUYhpFY1LrOnjdvHv3792f8+PFcfvnl+7jOHjZsGKNHj+awww7jmmuuabJHcOeddzJ79uxG85x22mmceOKJ+6SlpKTwyiuv8Otf/5qRI0cyatQoPvvsMwDuvfdejjzySI499liGDo3tKkeJRvejyUpFkoG3gPdU9eF6zvcD3lLVwxorZ+zYsXogu0Z3LvuEtVddQ+6Z58AXy6lavx7xetGqKjJPnsRVo79im7eMPz5RTY9Boxj+7xcirqO1MW3aNMDp1hpNY/qqn5UrV3LooYfWey4WkdfaIg3ovMEJnJj3FMQJzPwUsDLUIIhI95BsZwNRcxyTntKVboeMoeqZV6netIne//gHQxYvosstN1M2dx6//I+XYRt89CpUuh41MVpiGIZhJBzxWH10LHApsExElrhptwEXicgonGWqm4BroiVA2pDB9H3+OarWrgUIxlTudOWVeNLT4e57uGOll9RBh9DhkkuiJYZhGEbCEY/VR/Opv+sS0z0JsNcYhNL+wgspX7iIPZ99Rq+//Q1vZkasxUpILjHjGBGmr8jp2LFjvEUwsCA7+yEi9PjTH9HKSjzt2sVbnIQhOTk53iK0KExfDaOqOKPI++LxmIOF5uZA5oztV6gHETGDUIcvv/ySL7/8Mt5itBhMX/WTlpZGUVFRvY2Vuc5uXlSVoqIi0tLSIrrOegpGWNSunx43blycJWkZmL7qp1evXuTl5VHfHqOysjIAMltxBMNYk5aWFtyFHS5mFAzDiBnJycn079+/3nO2jDcxsOEjwzAMI4gZBcMwDCOIGQXDMAwjSFzcXDQXIlIAbD6ASzsDhc0sTnNgckVOospmckVGosoFiSvbwchVqKqn1neiRRuFA0VEFqrq2HjLUReTK3ISVTaTKzISVS5IXNmiJZcNHxmGYRhBzCgYhmEYQdqqUXgi3gI0gMkVOYkqm8kVGYkqFySubFGRq03OKRiGYRj101Z7CoZhGEY9tDmjICKnishqEVknIrfGUY7eIvKxiHwjIitE5Gdu+l0iskVElrif0+Mg2yYRWebWv9BN6ygiH4jIWvdvvTG0oyjTkBCdLBGREhG5KV76cuOI7xCR5SFp9epIHB5zn7mlIjI6xnL9UURWuXW/LiLt3fR+IlIRort/xFiuBn87EfmNq6/VIjI5xnLNDJFpU23clxjrq6H2IfrPmKq2mQ/gBdYDA4AU4GtgWJxk6Q6Mdo+zgDXAMOAu4OY462kT0LlO2oPAre7xrcADcf4dtwF946Uv4HhgNLC8KR0BpwPv4MQROQr4X4zlOgVIco8fCJGrX2i+OOir3t/O/T/4GkgF+rv/s95YyVXn/EPAnXHQV0PtQ9SfsbbWUxgPrFPVDapaDcwAzoyHIKqar6qL3eNSYCXQMx6yhMmZwDPu8TPAWfEThZOA9ap6IBsXmwVV/S+ws05yQzo6E3hWHb4A2tcJPxtVuVT1fVWtjT7/BRCZ28woydUIZwIzVLVKVTcC63D+d2Mqlxs6+ALgxWjU3RiNtA9Rf8bamlHoCXwX8j2PBGiIRaQfcATwPzfpercL+HSsh2lcFHhfRBaJyNVuWldVzXePtwFd4yBXLRey7z9qvPVVS0M6SqTn7kc4b5S19BeRr0RknohMiIM89f12iaKvCcB2VV0bkhZzfdVpH6L+jLU1o5BwiEgm8Cpwk6qWAI8DA4FRQD5O9zXWHKeqo4HTgOtE5PjQk+r0V+OybE1EUoApwMtuUiLoaz/iqaOGEJHbgRpgupuUD/RR1SOAXwAviEh2DEVKyN8uhIvY9+Uj5vqqp30IEq1nrK0ZhS1A75Dvvdy0uCAiyTg/+HRVfQ1AVberql9VA8CTRKnb3BiqusX9uwN43ZVhe2131P27I9ZyuZwGLFbV7a6McddXCA3pKO7PnYhMBb4PXOI2JrjDM0Xu8SKcsfvBsZKpkd8uEfSVBJwDzKxNi7W+6msfiMEz1taMwpfAIBHp775xXgjMjocg7njlU8BKVX04JD10HPBsYHnda6MsV4aIZNUe40xSLsfR0+VutsuBN2IpVwj7vL3FW191aEhHs4HL3BUiRwHFIUMAUUdETgV+BUxR1fKQ9FwR8brHA4BBwIYYytXQbzcbuFBEUkWkvyvXgljJ5TIJWKWqebUJsdRXQ+0DsXjGYjGTnkgfnFn6NThW/vY4ynEcTtdvKbDE/ZwOPAcsc9NnA91jLNcAnJUfXwMranUEdALmAGuBD4GOcdBZBlAE5ISkxUVfOIYpH/DhjN9e2ZCOcFaE/M195pYBY2Ms1zqc8eba5+wfbt5z3d94CbAYOCPGcjX42wG3u/paDZwWS7nc9GnAtXXyxlJfDbUPUX/GbEezYRiGEaStDR8ZhmEYjWBGwTAMwwhiRsEwDMMIYkbBMAzDCGJGwTAMwwhiRsGIOSKiIvJQyPebReSuZip7moic1xxlNVHP+SKyUkQ+rufcIBF5S0TWu65CPq67KzyWiMhZIjIs5Ps9IjIpXvIYiY0ZBSMeVAHniEjneAsSiruLNVyuBK5S1RPrlJEG/Ad4QlUHquoY4Aac/R9Ro3ZTVQOcheNhEwBVvVNVP4ymPEbLxYyCEQ9qcEIJ/rzuibpv+iJS5v6d6Dohe0NENojI/SJyiYgsECf2w8CQYiaJyEIRWSMi33ev94oTV+BL1wHbNSHlfiIis4Fv6pHnIrf85SLygJt2J87moqdE5I91LrkE+FxVgzvlVXW5qk5zr81wnb8tcB2rnemmTxWR10TkXXF85T8YIsMpIvK5iCwWkZddfzi1cS8eEJHFwPkicpV7f1+LyKsiki4ix+D4ivqjODEABobqWEROcuVY5sqVGlL23W6dy0RkqJt+guyNJ/BV7e53o/VgRsGIF38DLhGRnAiuGQlcCxwKXAoMVtXxwL9w3sZr6YfjR+f/Af9w396vxNn6Pw4YB1zlulAAx5/+z1R1Hz82ItIDJ/7A93Ccto0TkbNU9R5gIY4foVvqyDgcZ7drQ9wOfOTKfSJOY53hnhsF/AAYAfxAnEArnYE7gEnqOClciOOMrZYiVR2tqjOA11R1nKqOxHG1fKWqfoazW/gWVR2lqutD7i8NZ+fuD1R1BJAE/CSk7EK3zseBm920m4HrVHUUjhfRikbu1WiBmFEw4oI6Hh+fBW6M4LIv1fEzX4Wznf99N30ZjiGo5SVVDajj8ngDMBTHh9Nl4kTR+h+Ou4BBbv4F6vjtr8s4YK6qFqgTj2A6TlCWsBEn0tlyEal1aHYKcKsrx1wgDejjnpujqsWqWonTa+mLEzBlGPCpe83lbnotM0OOD3N7PctweizDmxBvCLBRVde435+pc3+1Mi9ir34/BR4WkRuB9ro3ToPRSohkDNUwmptHcN6q/x2SVoP7siIiHpwIebVUhRwHQr4H2PdZruu7RXF8w9ygqu+FnhCRicCeAxG+AVYQ0rCq6tkiMhb4U22VwLmqurqOHEey7/35ce5JgA9U9aIG6guVfRpwlqp+LY5X1IkHfhsQIk+tLKjq/SLyHxw/PJ+KyGRVXXWQ9RgJhPUUjLihqjuBl3CGdmrZBIxxj6cAyQdQ9Pki4nHnGQbgOFV7D/iJOO6IEZHBIcM2DbEAOEFEOrsTuRcB85q45gXgWBGZEpKWHnL8HnCDiIgrxxFNlPeFW94hbv4MEWnIXXMWkO/e4yUh6aXuubqsBvrVlo0zJNfo/YnIQFVdpqoP4HgdHtqE/EYLw4yCEW8eAkJXIT2J0xB/DRzNgb3Ff4vToL+D4+myEmfe4RtgsThB2v9JEz1ldVwP3wp8jOM1dpGqNuoyXFUrcOIWXOtOiH+OMyfwezfLvTiGbqmIrHC/N1ZeATAVeFFElgKf03BD/FucobFPgdC39xnALe7EcHBC3tXLFcDL7pBTAGgqGP1N7nDYUhzPou80kd9oYZiXVMMwDCOI9RQMwzCMIGYUDMMwjCBmFAzDMIwgZhQMwzCMIGYUDMMwjCBmFAzDMIwgZhQMwzCMIGYUDMMwjCD/HxE4k+kJNvzvAAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -234,16 +244,16 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "WrapperParams(strategy_params=EvoParams(mu_eff=DeviceArray(26.966648, dtype=float32), c_1=DeviceArray(1.2144131e-06, dtype=float32), c_mu=DeviceArray(3.0331763e-05, dtype=float32), c_sigma=DeviceArray(0.02204519, dtype=float32), d_sigma=DeviceArray(1.0220451, dtype=float32), c_c=DeviceArray(0.00312667, dtype=float32), chi_n=DeviceArray(35.798042, dtype=float32), c_m=1.0, sigma_init=0.065, init_min=0.0, init_max=0.0, clip_min=-3.4028235e+38, clip_max=3.4028235e+38), restart_params=RestartParams(min_num_gens=50, min_fitness_spread=1, popsize_multiplier=2, tol_x=1e-12, tol_x_up=10000.0, tol_condition_C=100000000000000.0))" + "WrapperParams(strategy_params=EvoParams(mu_eff=DeviceArray(26.966648, dtype=float32), c_1=DeviceArray(1.2144133e-06, dtype=float32), c_mu=DeviceArray(3.0331763e-05, dtype=float32), c_sigma=DeviceArray(0.02204519, dtype=float32), d_sigma=DeviceArray(1.0220451, dtype=float32), c_c=DeviceArray(0.00312667, dtype=float32), chi_n=DeviceArray(35.798046, dtype=float32, weak_type=True), c_m=1.0, sigma_init=1.0, init_min=0.0, init_max=0.0, clip_min=-3.4028235e+38, clip_max=3.4028235e+38), restart_params=RestartParams(min_num_gens=50, min_fitness_spread=1, popsize_multiplier=2, tol_x=1e-12, tol_x_up=10000.0, tol_condition_C=100000000000000.0, copy_mean=True))" ] }, - "execution_count": 31, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -265,12 +275,20 @@ "cell_type": "code", "execution_count": 8, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ParameterReshaper: 1282 parameters detected for optimization.\n" + ] + } + ], "source": [ "# Use single device due to odd/even popsizes -> makes it hard to make sure even division\n", "param_reshaper = ParameterReshaper(params, n_devices=1)\n", - "evaluator = GymFitness(\"CartPole-v1\", num_env_steps=200, num_rollouts=16, n_devices=1)\n", - "evaluator.set_apply_fn(param_reshaper.vmap_dict, network.apply)" + "evaluator = GymnaxFitness(\"CartPole-v1\", num_env_steps=200, num_rollouts=16, n_devices=1)\n", + "evaluator.set_apply_fn(network.apply)" ] }, { @@ -278,44 +296,53 @@ "execution_count": 9, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/rob/anaconda3/envs/mle-toolbox/lib/python3.9/site-packages/flax/core/scope.py:740: FutureWarning: jax.tree_leaves is deprecated, and will be removed in a future release. Use jax.tree_util.tree_leaves instead.\n", + " abs_value_flat = jax.tree_leaves(abs_value)\n", + "/Users/rob/anaconda3/envs/mle-toolbox/lib/python3.9/site-packages/flax/core/scope.py:741: FutureWarning: jax.tree_leaves is deprecated, and will be removed in a future release. Use jax.tree_util.tree_leaves instead.\n", + " value_flat = jax.tree_leaves(value)\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "Generation: 0 Perf (Best): 24.1875 Perf (Mean): 21.789375\n", - "Generation: 20 Perf (Best): 29.5625 Perf (Mean): 20.15625\n", - "Generation: 40 Perf (Best): 29.5625 Perf (Mean): 21.512499\n", - "Generation: 60 Perf (Best): 31.5625 Perf (Mean): 17.129375\n", - "Generation: 80 Perf (Best): 33.75 Perf (Mean): 24.57625\n", - "Generation: 100 Perf (Best): 53.4375 Perf (Mean): 31.818748\n", - "Generation: 120 Perf (Best): 130.8125 Perf (Mean): 76.581245\n", - "Generation: 140 Perf (Best): 200.0 Perf (Mean): 171.45375\n", - "Generation: 160 Perf (Best): 200.0 Perf (Mean): 199.22499\n", - "--> Restarted Strategy: Gen 180\n", - "--> New Popsize: 200\n", - "Generation: 180 Perf (Best): 200.0 Perf (Mean): 19.42125\n", - "Generation: 200 Perf (Best): 200.0 Perf (Mean): 20.525936\n", - "Generation: 220 Perf (Best): 200.0 Perf (Mean): 20.189062\n", - "Generation: 240 Perf (Best): 200.0 Perf (Mean): 23.519999\n", - "Generation: 260 Perf (Best): 200.0 Perf (Mean): 21.384375\n", - "Generation: 280 Perf (Best): 200.0 Perf (Mean): 21.339375\n", - "Generation: 300 Perf (Best): 200.0 Perf (Mean): 25.099375\n", - "Generation: 320 Perf (Best): 200.0 Perf (Mean): 29.201874\n", - "Generation: 340 Perf (Best): 200.0 Perf (Mean): 30.66\n", - "Generation: 360 Perf (Best): 200.0 Perf (Mean): 29.360624\n", - "Generation: 380 Perf (Best): 200.0 Perf (Mean): 42.391872\n", - "Generation: 400 Perf (Best): 200.0 Perf (Mean): 59.089687\n", - "Generation: 420 Perf (Best): 200.0 Perf (Mean): 70.78906\n", - "Generation: 440 Perf (Best): 200.0 Perf (Mean): 82.59062\n", - "Generation: 460 Perf (Best): 200.0 Perf (Mean): 117.78406\n", - "Generation: 480 Perf (Best): 200.0 Perf (Mean): 171.02657\n", - "Generation: 500 Perf (Best): 200.0 Perf (Mean): 173.53874\n", - "Generation: 520 Perf (Best): 200.0 Perf (Mean): 194.27187\n", - "Generation: 540 Perf (Best): 200.0 Perf (Mean): 197.54688\n", - "Generation: 560 Perf (Best): 200.0 Perf (Mean): 199.68718\n", - "--> Restarted Strategy: Gen 571\n", - "--> New Popsize: 283\n", - "Generation: 580 Perf (Best): 200.0 Perf (Mean): 19.240063\n" + "Generation: 0 Perf (Best): 149.4375 Perf (Mean): 16.563124\n", + "Generation: 20 Perf (Best): 200.0 Perf (Mean): 49.830624\n", + "Generation: 40 Perf (Best): 200.0 Perf (Mean): 178.595\n", + "Generation: 60 Perf (Best): 200.0 Perf (Mean): 195.30812\n", + "Generation: 80 Perf (Best): 200.0 Perf (Mean): 194.295\n", + "Generation: 100 Perf (Best): 200.0 Perf (Mean): 196.60187\n", + "Generation: 120 Perf (Best): 200.0 Perf (Mean): 186.63875\n", + "Generation: 140 Perf (Best): 200.0 Perf (Mean): 187.41937\n", + "Generation: 160 Perf (Best): 200.0 Perf (Mean): 196.69\n", + "Generation: 180 Perf (Best): 200.0 Perf (Mean): 182.97624\n", + "Generation: 200 Perf (Best): 200.0 Perf (Mean): 175.15187\n", + "Generation: 220 Perf (Best): 200.0 Perf (Mean): 178.8075\n", + "Generation: 240 Perf (Best): 200.0 Perf (Mean): 182.48563\n", + "Generation: 260 Perf (Best): 200.0 Perf (Mean): 190.94063\n", + "Generation: 280 Perf (Best): 200.0 Perf (Mean): 181.2175\n", + "Generation: 300 Perf (Best): 200.0 Perf (Mean): 155.97063\n", + "Generation: 320 Perf (Best): 200.0 Perf (Mean): 150.36\n", + "Generation: 340 Perf (Best): 200.0 Perf (Mean): 116.10562\n", + "Generation: 360 Perf (Best): 200.0 Perf (Mean): 64.515\n", + "Generation: 380 Perf (Best): 200.0 Perf (Mean): 25.566874\n", + "Generation: 400 Perf (Best): 200.0 Perf (Mean): 23.734999\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/var/folders/y4/1lwbxdz55wzg_83326j5cjk40000gn/T/ipykernel_41931/3292922320.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 29\u001b[0m \u001b[0mfitness\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevaluator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrng_eval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreshaped_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 30\u001b[0m \u001b[0mfit_re\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfit_shaper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfitness\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m \u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstrategy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfit_re\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mes_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 32\u001b[0m \u001b[0mlog\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mes_logging\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlog\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfitness\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/anaconda3/envs/mle-toolbox/lib/python3.9/site-packages/flax/struct.py\u001b[0m in \u001b[0;36mclz_from_iterable\u001b[0;34m(meta, data)\u001b[0m\n\u001b[1;32m 118\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeta\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 120\u001b[0;31m \u001b[0;32mdef\u001b[0m \u001b[0mclz_from_iterable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmeta\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 121\u001b[0m \u001b[0mmeta_args\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmeta_fields\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeta\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 122\u001b[0m \u001b[0mdata_args\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_fields\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " ] } ], @@ -359,7 +386,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, "outputs": [ { diff --git a/examples/07_brax_control.ipynb b/examples/07_brax_control.ipynb index 5ac4e46..dbd8ad9 100644 --- a/examples/07_brax_control.ipynb +++ b/examples/07_brax_control.ipynb @@ -20,7 +20,7 @@ "%config InlineBackend.figure_format = 'retina'\n", "\n", "!pip install -q git+https://github.com/RobertTLange/evosax.git@main\n", - "!pip install -q brax" + "!pip install -q brax evojax" ] }, { @@ -32,133 +32,103 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ParameterReshaper: 6248 parameters detected for optimization.\n" - ] - } - ], + "outputs": [], "source": [ - "import jax\n", - "import jax.numpy as jnp\n", - "\n", - "from evosax import OpenES, ParameterReshaper, FitnessShaper, NetworkMapper\n", - "from evosax.utils import ESLog\n", - "from evosax.problems import BraxFitness\n", + "from evojax.obs_norm import ObsNormalizer\n", + "from evojax.sim_mgr import SimManager\n", + "from evojax.task.brax_task import BraxTask\n", + "from evojax.policy import MLPPolicy\n", "\n", - "# Instantiate brax rollout wrapper & network architecture\n", - "evaluator = BraxFitness(\"ant\", num_env_steps=1000, num_rollouts=16)\n", - "\n", - "rng = jax.random.PRNGKey(0)\n", - "network = NetworkMapper[\"MLP\"](\n", - " num_hidden_units=32,\n", - " num_hidden_layers=4,\n", - " num_output_units=evaluator.action_shape,\n", - " hidden_activation=\"tanh\",\n", - " output_activation=\"tanh\",\n", - ")\n", - "pholder = jnp.zeros((1, evaluator.input_shape[0]))\n", - "params = network.init(\n", - " rng,\n", - " x=pholder,\n", - " rng=rng,\n", - ")\n", - "\n", - "param_reshaper = ParameterReshaper(params)\n", - "\n", - "# Set mapping dictionary for parallelization\n", - "evaluator.set_apply_fn(param_reshaper.vmap_dict, network.apply)" + "from evosax import Strategies\n", + "from evosax.utils.evojax_wrapper import Evosax2JAX_Wrapper" ] }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "EvoParams(opt_params=OptParams(lrate_init=0.01, lrate_decay=0.999, lrate_limit=0.001, momentum=0.9, beta_1=None, beta_2=None, eps=None, max_speed=None), sigma_init=0.04, sigma_decay=0.999, sigma_limit=0.01, init_min=0.0, init_max=0.0, clip_min=-3.4028235e+38, clip_max=3.4028235e+38)" - ] - }, - "execution_count": 17, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "strategy = OpenES(popsize=256,\n", - " num_dims=param_reshaper.total_params,\n", - " opt_name=\"adam\")\n", - "strategy.default_params" + "def get_brax_task(\n", + " env_name = \"ant\",\n", + " hidden_dims = [32, 32, 32, 32],\n", + "):\n", + " train_task = BraxTask(env_name, test=False)\n", + " test_task = BraxTask(env_name, test=True)\n", + " policy = MLPPolicy(\n", + " input_dim=train_task.obs_shape[0],\n", + " output_dim=train_task.act_shape[0],\n", + " hidden_dims=hidden_dims,\n", + " )\n", + " return train_task, test_task, policy" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Generation: 0 Generation: 203.22612\n", - "Generation: 20 Generation: 203.62665\n" - ] - }, - { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m/var/folders/y4/1lwbxdz55wzg_83326j5cjk40000gn/T/ipykernel_75476/1777371750.py\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstrategy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrng_ask\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 20\u001b[0m \u001b[0mreshaped_params\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparam_reshaper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m \u001b[0mfitness\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevaluator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrollout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrng_eval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreshaped_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 22\u001b[0m \u001b[0mfit_re\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfit_shaper\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfitness\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 23\u001b[0m \u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstrategy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtell\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfit_re\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Dropbox/core-code/develop-jax/evosax/evosax/problems/control_brax.py\u001b[0m in \u001b[0;36mrollout\u001b[0;34m(self, rng_input, policy_params)\u001b[0m\n\u001b[1;32m 107\u001b[0m \u001b[0;34m\"\"\"Placeholder fn call for rolling out a population for multi-evals.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 108\u001b[0m \u001b[0mrng_pop\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mjax\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrng_input\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnum_rollouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 109\u001b[0;31m scores, all_obs, masks = jax.jit(self.rollout_map)(\n\u001b[0m\u001b[1;32m 110\u001b[0m \u001b[0mrng_pop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpolicy_params\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 111\u001b[0m )\n", - " \u001b[0;31m[... skipping hidden 1 frame]\u001b[0m\n", - "\u001b[0;32m~/anaconda3/envs/mle-toolbox/lib/python3.9/site-packages/jax/_src/api.py\u001b[0m in \u001b[0;36mcache_miss\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 471\u001b[0m \u001b[0min_type\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfer_lambda_input_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs_flat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 472\u001b[0m \u001b[0mflat_fun\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlu\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mannotate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mflat_fun\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0min_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 473\u001b[0;31m out_flat = xla.xla_call(\n\u001b[0m\u001b[1;32m 474\u001b[0m \u001b[0mflat_fun\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs_flat\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 475\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbackend\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbackend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mflat_fun\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/envs/mle-toolbox/lib/python3.9/site-packages/jax/core.py\u001b[0m in \u001b[0;36mbind\u001b[0;34m(self, fun, *args, **params)\u001b[0m\n\u001b[1;32m 1763\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1764\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mbind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfun\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1765\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mcall_bind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfun\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1766\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1767\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_bind_params\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/envs/mle-toolbox/lib/python3.9/site-packages/jax/core.py\u001b[0m in \u001b[0;36mcall_bind\u001b[0;34m(primitive, fun, *args, **params)\u001b[0m\n\u001b[1;32m 1779\u001b[0m \u001b[0mtracers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtop_trace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfull_raise\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1780\u001b[0m \u001b[0mfun_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlu\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mannotate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfun_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfun\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0min_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1781\u001b[0;31m \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtop_trace\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocess_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprimitive\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfun_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtracers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1782\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfull_lower\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mapply_todos\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0menv_trace_todo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mouts\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1783\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/envs/mle-toolbox/lib/python3.9/site-packages/jax/core.py\u001b[0m in \u001b[0;36mprocess_call\u001b[0;34m(self, primitive, f, tracers, params)\u001b[0m\n\u001b[1;32m 676\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 677\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mprocess_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprimitive\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtracers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 678\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mprimitive\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimpl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mtracers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 679\u001b[0m \u001b[0mprocess_map\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprocess_call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 680\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/envs/mle-toolbox/lib/python3.9/site-packages/jax/_src/dispatch.py\u001b[0m in \u001b[0;36m_xla_call_impl\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m 183\u001b[0m keep_unused, *arg_specs)\n\u001b[1;32m 184\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 185\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mcompiled_fun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 186\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mFloatingPointError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 187\u001b[0m \u001b[0;32massert\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjax_debug_nans\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjax_debug_infs\u001b[0m \u001b[0;31m# compiled_fun can only raise in this case\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/anaconda3/envs/mle-toolbox/lib/python3.9/site-packages/jax/_src/dispatch.py\u001b[0m in \u001b[0;36m_execute_compiled\u001b[0;34m(name, compiled, input_handler, output_buffer_counts, result_handlers, effects, kept_var_idx, *args)\u001b[0m\n\u001b[1;32m 613\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0meffects\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 614\u001b[0m \u001b[0minput_bufs_flat\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtoken_handler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_add_tokens\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0meffects\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_bufs_flat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 615\u001b[0;31m \u001b[0mout_bufs_flat\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompiled\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_bufs_flat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 616\u001b[0m \u001b[0mcheck_special\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout_bufs_flat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 617\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_buffer_counts\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + "ParameterReshaper: 6248 parameters detected for optimization.\n" ] } ], "source": [ - "num_generations = 1000\n", - "print_every_k_gens = 20\n", - "\n", - "es_logging = ESLog(param_reshaper.total_params,\n", - " num_generations,\n", - " top_k=5,\n", - " maximize=True)\n", - "log = es_logging.initialize()\n", - "\n", - "fit_shaper = FitnessShaper(centered_rank=True,\n", - " z_score=True,\n", - " w_decay=0.1,\n", - " maximize=True)\n", - "\n", - "state = strategy.initialize(rng)\n", + "train_task, test_task, policy = get_brax_task(\"ant\")\n", + "solver = Evosax2JAX_Wrapper(\n", + " Strategies[\"OpenES\"],\n", + " param_size=policy.num_params,\n", + " pop_size=256,\n", + " es_config={\"maximize\": True,\n", + " \"centered_rank\": True,\n", + " \"lrate_init\": 0.01,\n", + " \"lrate_decay\": 0.999,\n", + " \"lrate_limit\": 0.001},\n", + " es_params={\"sigma_init\": 0.05,\n", + " \"sigma_decay\": 0.999,\n", + " \"sigma_limit\": 0.01},\n", + " seed=0,\n", + ")\n", + "obs_normalizer = ObsNormalizer(\n", + " obs_shape=train_task.obs_shape, dummy=not True\n", + ")\n", + "sim_mgr = SimManager(\n", + " policy_net=policy,\n", + " train_vec_task=train_task,\n", + " valid_vec_task=test_task,\n", + " seed=0,\n", + " obs_normalizer=obs_normalizer,\n", + " pop_size=256,\n", + " use_for_loop=False,\n", + " n_repeats=16,\n", + " test_n_repeats=1,\n", + " n_evaluations=128\n", + ")\n", "\n", - "for gen in range(num_generations):\n", - " rng, rng_init, rng_ask, rng_eval = jax.random.split(rng, 4)\n", - " x, state = strategy.ask(rng_ask, state)\n", - " reshaped_params = param_reshaper.reshape(x)\n", - " fitness = evaluator.rollout(rng_eval, reshaped_params).mean(axis=1)\n", - " fit_re = fit_shaper.apply(x, fitness)\n", - " state = strategy.tell(x, fit_re, state)\n", - " log = es_logging.update(log, x, fitness)\n", - " \n", - " if gen % print_every_k_gens == 0:\n", - " print(\"Generation: \", gen, \"Generation: \", log[\"log_top_1\"][gen])" + "print(f\"START EVOLVING {policy.num_params} PARAMS.\")\n", + "# Run ES Loop.\n", + "for gen_counter in range(1500):\n", + " params = solver.ask()\n", + " scores, _ = sim_mgr.eval_params(params=params, test=False)\n", + " solver.tell(fitness=scores)\n", + " if gen_counter == 0 or (gen_counter + 1) % 50 == 0:\n", + " test_scores, _ = sim_mgr.eval_params(\n", + " params=solver.best_params, test=True\n", + " )\n", + " print(\n", + " {\n", + " \"num_gens\": gen_counter + 1,\n", + " },\n", + " {\n", + " \"train_perf\": float(np.nanmean(scores)),\n", + " \"test_perf\": float(np.nanmean(test_scores)),\n", + " },\n", + " )" ] }, { @@ -168,16 +138,6 @@ "# Visualize Learning Curve and Policy" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Plot the learning curve over generations\n", - "es_logging.plot(log, \"Ant MLP OpenAI-ES\")" - ] - }, { "cell_type": "code", "execution_count": 31, diff --git a/examples/08_encodings.ipynb b/examples/08_encodings.ipynb deleted file mode 100644 index eb82481..0000000 --- a/examples/08_encodings.ipynb +++ /dev/null @@ -1,276 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 08 - Indirect Encodings\n", - "### [Last Update: June 2022][![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/08_encodings.ipynb)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%matplotlib inline\n", - "%load_ext autoreload\n", - "%autoreload 2\n", - "%config InlineBackend.figure_format = 'retina'\n", - "\n", - "!pip install -q git+https://github.com/RobertTLange/evosax.git@main\n", - "!pip install -q gymnax" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Experimental (!!!) - Random Encodings" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - ":228: RuntimeWarning: scipy._lib.messagestream.MessageStream size changed, may indicate binary incompatibility. Expected 56 from C header, got 64 from PyObject\n", - "WARNING:absl:No GPU/TPU found, falling back to CPU. (Set TF_CPP_MIN_LOG_LEVEL=0 and rerun for more info.)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ParameterReshaper: 4610 parameters detected for optimization.\n" - ] - }, - { - "data": { - "text/plain": [ - "DeviceArray(4610, dtype=int32)" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import jax\n", - "import jax.numpy as jnp\n", - "from evosax import NetworkMapper\n", - "from evosax.problems import GymFitness\n", - "from evosax.utils import ParameterReshaper\n", - "\n", - "rng = jax.random.PRNGKey(0)\n", - "# Run Strategy on CartPole MLP\n", - "evaluator = GymFitness(\"CartPole-v1\", num_env_steps=200, num_rollouts=16)\n", - "\n", - "network = NetworkMapper[\"MLP\"](\n", - " num_hidden_units=64,\n", - " num_hidden_layers=2,\n", - " num_output_units=2,\n", - " hidden_activation=\"relu\",\n", - " output_activation=\"categorical\",\n", - ")\n", - "pholder = jnp.zeros((1, evaluator.input_shape[0]))\n", - "params = network.init(\n", - " rng,\n", - " x=pholder,\n", - " rng=rng,\n", - ")\n", - "\n", - "reshaper = ParameterReshaper(params)\n", - "reshaper.total_params" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "from evosax.utils import FitnessShaper\n", - "from evosax.experimental.decodings import RandomDecoder\n", - "\n", - "# Only optimize 10 parameters!\n", - "num_encoding_dims = 6\n", - "reshaper = RandomDecoder(num_encoding_dims, params)\n", - "evaluator.set_apply_fn(reshaper.vmap_dict, network.apply)\n", - "\n", - "fit_shaper = FitnessShaper(maximize=True)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/rob/anaconda3/envs/mle-toolbox/lib/python3.9/site-packages/jax/_src/tree_util.py:188: FutureWarning: jax.tree_util.tree_multimap() is deprecated. Please use jax.tree_util.tree_map() instead as a drop-in replacement.\n", - " warnings.warn('jax.tree_util.tree_multimap() is deprecated. Please use jax.tree_util.tree_map() '\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "20 133.08125 200.0 50.5719 -200.0\n", - "40 149.9775 200.0 45.9242 -200.0\n", - "60 157.07562 200.0 51.393032 -200.0\n", - "80 151.68312 200.0 53.497288 -200.0\n", - "100 160.70312 200.0 50.2572 -200.0\n" - ] - } - ], - "source": [ - "from evosax import DE\n", - "\n", - "strategy = DE(\n", - " num_dims=reshaper.total_params,\n", - " popsize=100,\n", - ")\n", - "state = strategy.initialize(rng)\n", - "\n", - "for t in range(100):\n", - " rng, rng_eval, rng_iter = jax.random.split(rng, 3)\n", - " x, state = strategy.ask(rng_iter, state)\n", - " x_re = reshaper.reshape(x)\n", - " fitness = evaluator.rollout(rng_eval, x_re).mean(axis=1)\n", - " fit_re = fit_shaper.apply(x, fitness)\n", - " state = strategy.tell(x, fit_re, state)\n", - "\n", - " if (t + 1) % 20 == 0:\n", - " print(\n", - " t + 1,\n", - " fitness.mean(),\n", - " fitness.max(),\n", - " fitness.std(),\n", - " state.best_fitness,\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Experimental (!!!) - Hypernetwork Encodings" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ParameterReshaper: 2306 parameters detected for optimization.\n" - ] - }, - { - "data": { - "text/plain": [ - "DeviceArray(2306, dtype=int32)" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from evosax.experimental.decodings import HyperDecoder\n", - "\n", - "reshaper = HyperDecoder(\n", - " params,\n", - " hypernet_config={\n", - " \"num_latent_units\": 3, # Latent units per module kernel/bias\n", - " \"num_hidden_units\": 2, # Hidden dimensionality of a_i^j embedding\n", - " },\n", - " )\n", - "reshaper.total_params" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "20 19.089375 30.3125 6.1910863 -33.4375\n", - "40 29.787498 195.5625 32.49928 -200.0\n", - "60 31.540625 200.0 44.170444 -200.0\n", - "80 28.501875 200.0 47.56071 -200.0\n", - "100 28.136875 200.0 44.376225 -200.0\n" - ] - } - ], - "source": [ - "strategy = DE(\n", - " num_dims=reshaper.total_params,\n", - " popsize=100,\n", - ")\n", - "state = strategy.initialize(rng)\n", - "\n", - "for t in range(100):\n", - " rng, rng_eval, rng_iter = jax.random.split(rng, 3)\n", - " x, state = strategy.ask(rng_iter, state)\n", - " x_re = reshaper.reshape(x)\n", - " fitness = evaluator.rollout(rng_eval, x_re).mean(axis=1)\n", - " fit_re = fit_shaper.apply(x, fitness)\n", - " state = strategy.tell(x, fit_re, state)\n", - "\n", - " if (t + 1) % 20 == 0:\n", - " print(\n", - " t + 1,\n", - " fitness.mean(),\n", - " fitness.max(),\n", - " fitness.std(),\n", - " state.best_fitness\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "mle-toolbox", - "language": "python", - "name": "mle-toolbox" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.7" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/examples/09_exp_batch_es.ipynb b/examples/09_exp_batch_es.ipynb deleted file mode 100644 index 450369f..0000000 --- a/examples/09_exp_batch_es.ipynb +++ /dev/null @@ -1,367 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# 09 - Batch Strategy Rollouts\n", - "### [Last Update: June 2022][![Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/RobertTLange/evosax/blob/main/examples/09_exp_batch_es.ipynb)" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "%matplotlib inline\n", - "%load_ext autoreload\n", - "%autoreload 2\n", - "%config InlineBackend.figure_format = 'retina'\n", - "\n", - "!pip install -q git+https://github.com/RobertTLange/evosax.git@main\n", - "!pip install -q gymnax" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Experimental (!!!) - Subpopulation Batch ES Rollouts" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - ":228: RuntimeWarning: scipy._lib.messagestream.MessageStream size changed, may indicate binary incompatibility. Expected 56 from C header, got 64 from PyObject\n", - "WARNING:absl:No GPU/TPU found, falling back to CPU. (Set TF_CPP_MIN_LOG_LEVEL=0 and rerun for more info.)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ParameterReshaper: 4610 parameters detected for optimization.\n" - ] - } - ], - "source": [ - "import jax\n", - "import jax.numpy as jnp\n", - "from evosax import NetworkMapper\n", - "from evosax.problems import GymFitness\n", - "from evosax.utils import ParameterReshaper, FitnessShaper\n", - "\n", - "rng = jax.random.PRNGKey(0)\n", - "# Run Strategy on CartPole MLP\n", - "evaluator = GymFitness(\"CartPole-v1\", num_env_steps=200, num_rollouts=16)\n", - "\n", - "network = NetworkMapper[\"MLP\"](\n", - " num_hidden_units=64,\n", - " num_hidden_layers=2,\n", - " num_output_units=2,\n", - " hidden_activation=\"relu\",\n", - " output_activation=\"categorical\",\n", - ")\n", - "pholder = jnp.zeros((1, evaluator.input_shape[0]))\n", - "params = network.init(\n", - " rng,\n", - " x=pholder,\n", - " rng=rng,\n", - ")\n", - "\n", - "reshaper = ParameterReshaper(params)\n", - "evaluator.set_apply_fn(reshaper.vmap_dict, network.apply)\n", - "\n", - "fit_shaper = FitnessShaper(maximize=True)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "from evosax.experimental.subpops import BatchStrategy\n", - "\n", - "strategy = BatchStrategy(\n", - " strategy_name=\"DE\",\n", - " num_dims=reshaper.total_params,\n", - " popsize=100,\n", - " num_subpops=5,\n", - " communication=\"best_subpop\",\n", - ")\n", - "params = strategy.default_params\n", - "state = strategy.initialize(rng, params)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1 22.464375 26.3125 2.0493376 [-26.3125 -26.3125 -26.3125 -26.3125 -26.3125]\n", - "2 22.575624 29.75 2.9526908 [-29.75 -29.75 -29.75 -29.75 -29.75]\n", - "3 22.914999 29.125 3.4180415 [-29.75 -29.75 -29.75 -29.75 -29.75]\n", - "4 19.238125 28.9375 2.5196369 [-29.75 -29.75 -29.75 -29.75 -29.75]\n", - "5 19.704374 33.0625 2.316076 [-33.0625 -33.0625 -33.0625 -33.0625 -33.0625]\n", - "6 23.7925 61.875 9.7088585 [-61.875 -61.875 -61.875 -61.875 -61.875]\n", - "7 35.21 118.5625 16.621597 [-118.5625 -118.5625 -118.5625 -118.5625 -118.5625]\n", - "8 38.021873 86.375 18.679571 [-118.5625 -118.5625 -118.5625 -118.5625 -118.5625]\n", - "9 45.83875 148.75 31.13269 [-148.75 -148.75 -148.75 -148.75 -148.75]\n", - "10 36.0625 125.6875 28.167828 [-148.75 -148.75 -148.75 -148.75 -148.75]\n", - "11 44.895 182.9375 38.524178 [-182.9375 -182.9375 -182.9375 -182.9375 -182.9375]\n", - "12 49.030624 170.0 36.70624 [-182.9375 -182.9375 -182.9375 -182.9375 -182.9375]\n", - "13 47.264374 170.75 32.65505 [-182.9375 -182.9375 -182.9375 -182.9375 -182.9375]\n", - "14 47.146248 174.8125 35.011383 [-182.9375 -182.9375 -182.9375 -182.9375 -182.9375]\n", - "15 57.025623 200.0 45.128643 [-200. -200. -200. -200. -200.]\n", - "16 87.83625 200.0 69.39789 [-200. -200. -200. -200. -200.]\n", - "17 73.627495 200.0 65.97414 [-200. -200. -200. -200. -200.]\n", - "18 75.694374 200.0 58.033886 [-200. -200. -200. -200. -200.]\n", - "19 73.02125 200.0 65.48465 [-200. -200. -200. -200. -200.]\n", - "20 82.159996 200.0 70.50161 [-200. -200. -200. -200. -200.]\n" - ] - } - ], - "source": [ - "for t in range(20):\n", - " rng, rng_eval, rng_iter = jax.random.split(rng, 3)\n", - " x, state = strategy.ask(rng_iter, state, params)\n", - " x_re = reshaper.reshape(x)\n", - " fitness = evaluator.rollout(rng_eval, x_re).mean(axis=1)\n", - " fit_re = fit_shaper.apply(x, fitness)\n", - " state = strategy.tell(x, fit_re, state, params)\n", - "\n", - " if t % 1 == 0:\n", - " print(\n", - " t + 1,\n", - " fitness.mean(),\n", - " fitness.max(),\n", - " fitness.std(),\n", - " state.best_fitness, # Best fitness in all subpops\n", - " )" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Experimental (!!!) - Subpopulation Meta-Batch ES Rollouts" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "EvoParams(mu_eff=DeviceArray(1.6496499, dtype=float32), c_1=DeviceArray(0.15949409, dtype=float32), c_mu=DeviceArray(0.02899084, dtype=float32), c_sigma=DeviceArray(0.42194194, dtype=float32), d_sigma=DeviceArray(1.421942, dtype=float32), c_c=DeviceArray(0.63072497, dtype=float32), chi_n=DeviceArray(1.2542727, dtype=float32), weights=DeviceArray([ 0.73042274, 0.2695773 , 0. , -0.726532 ,\n", - " -1.2900741 ], dtype=float32), weights_truncated=DeviceArray([0.73042274, 0.2695773 , 0. , 0. , 0. ], dtype=float32), c_m=1.0, sigma_init=0.065, init_min=DeviceArray([0.8, 0.9], dtype=float32), init_max=DeviceArray([0.8, 0.9], dtype=float32), clip_min=-3.4028235e+38, clip_max=3.4028235e+38)" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from evosax.experimental.subpops import MetaStrategy\n", - "\n", - "meta_strategy = MetaStrategy(\n", - " meta_strategy_name=\"CMA_ES\",\n", - " inner_strategy_name=\"DE\",\n", - " meta_params=[\"diff_w\", \"cross_over_rate\"],\n", - " num_dims=reshaper.total_params,\n", - " popsize=100,\n", - " num_subpops=5,\n", - " meta_strategy_kwargs={\"elite_ratio\": 0.5},\n", - " )\n", - "meta_es_params = meta_strategy.default_params_meta\n", - "meta_es_params.replace(\n", - " clip_min=jnp.array([0, 0]), clip_max=jnp.array([2, 1])\n", - ")\n", - "meta_es_params" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1 20.289999 23.5 1.0141777 [-22.125 -23.5 -23.5 -23.5 -22.125]\n", - "[0.879098 0.76078224 0.7285294 0.8667383 0.9170291 ]\n", - "[0.87730575 0.89257807 0.88649285 0.8704477 0.95789057]\n", - "====================\n", - "2 23.526875 27.8125 2.8445435 [-27.75 -27.75 -27.5625 -27.8125 -27.5625]\n", - "[0.7529136 0.75194293 0.75470865 0.73406416 0.7236362 ]\n", - "[0.92120713 0.80959445 0.904763 0.9089725 0.9178807 ]\n", - "====================\n", - "3 18.4025 23.375 1.7486227 [-27.75 -27.75 -27.5625 -27.8125 -27.5625]\n", - "[0.76000106 0.7763824 0.80698913 0.7326284 0.79374725]\n", - "[0.8445878 0.90652514 0.9469857 0.9224319 0.8527581 ]\n", - "====================\n", - "4 20.2475 28.875 2.2587662 [-27.75 -27.75 -27.5625 -28.875 -27.5625]\n", - "[0.8238988 0.7451631 0.7800138 0.8030797 0.7789296]\n", - "[0.81761867 0.87493116 0.85717034 0.8252281 0.8858736 ]\n", - "====================\n", - "5 21.651875 26.75 2.3301566 [-27.75 -27.75 -27.5625 -28.875 -27.5625]\n", - "[0.7483712 0.78120756 0.7842538 0.8036731 0.8382279 ]\n", - "[0.8533484 0.85126436 0.81118304 0.87271136 0.7978533 ]\n", - "====================\n", - "6 24.300625 32.5625 3.8705084 [-30.1875 -30.75 -28.5625 -32.5625 -29.625 ]\n", - "[0.7154043 0.73717016 0.74947 0.75264627 0.7836797 ]\n", - "[0.8599149 0.8408388 0.80433404 0.9227266 0.866442 ]\n", - "====================\n", - "7 22.035 33.8125 4.2435365 [-33.8125 -30.75 -28.875 -32.5625 -30.3125]\n", - "[0.73832107 0.7330418 0.80643374 0.7836552 0.74030274]\n", - "[0.8556646 0.80183303 0.82818526 0.83224803 0.84638065]\n", - "====================\n", - "8 22.17375 49.0 7.518158 [-45.5625 -42.0625 -28.875 -49. -45.4375]\n", - "[0.7866129 0.73554915 0.7203534 0.752935 0.69754535]\n", - "[0.84873676 0.8862246 0.83316815 0.79738003 0.8420979 ]\n", - "====================\n", - "9 23.22 70.8125 12.524904 [-60.6875 -70.8125 -28.875 -66.5 -45.4375]\n", - "[0.73071593 0.7588424 0.7373127 0.78221434 0.819327 ]\n", - "[0.8632609 0.85133994 0.88077104 0.8476571 0.82447743]\n", - "====================\n", - "10 27.818125 95.25 17.123335 [-60.6875 -70.8125 -28.875 -67.0625 -95.25 ]\n", - "[0.74657243 0.73996896 0.74008286 0.77336246 0.76170486]\n", - "[0.8573377 0.8602473 0.8592791 0.8566864 0.87690324]\n", - "====================\n", - "11 33.4875 170.375 33.512127 [-147.625 -149.6875 -28.875 -89.5625 -170.375 ]\n", - "[0.74657285 0.77310294 0.7667397 0.79113615 0.7745692 ]\n", - "[0.8604234 0.8604967 0.801528 0.87977314 0.8678907 ]\n", - "====================\n", - "12 41.148125 180.875 36.242935 [-147.625 -180.875 -28.875 -89.5625 -170.375 ]\n", - "[0.784878 0.7705825 0.7628718 0.7921372 0.7565861]\n", - "[0.8717209 0.84912974 0.8673709 0.847437 0.8717182 ]\n", - "====================\n", - "13 47.984375 200.0 39.957558 [-147.625 -200. -28.875 -89.5625 -180.5 ]\n", - "[0.76560074 0.76622653 0.7893174 0.77328974 0.7705352 ]\n", - "[0.8389833 0.852461 0.86471146 0.8482863 0.86811644]\n", - "====================\n", - "14 44.32375 200.0 47.0014 [-147.625 -200. -28.875 -94.5 -200. ]\n", - "[0.7761689 0.7649889 0.78513336 0.76904625 0.775042 ]\n", - "[0.85951185 0.854263 0.87067384 0.8597369 0.86205065]\n", - "====================\n", - "15 52.1825 200.0 46.55368 [-147.625 -200. -28.875 -94.5 -200. ]\n", - "[0.76948667 0.76638454 0.7677278 0.79087144 0.7603321 ]\n", - "[0.8666052 0.86009985 0.84476924 0.8650915 0.85453224]\n", - "====================\n", - "16 52.930622 200.0 51.577286 [-149.9375 -200. -28.875 -132.625 -200. ]\n", - "[0.75860137 0.77249414 0.75932413 0.76306224 0.7624783 ]\n", - "[0.8530121 0.8678422 0.8509262 0.85580117 0.8567307 ]\n", - "====================\n", - "17 51.984375 192.9375 48.96752 [-175.875 -200. -28.875 -132.625 -200. ]\n", - "[0.7763821 0.7706568 0.7801008 0.7734966 0.77879196]\n", - "[0.8634914 0.85994005 0.87226665 0.85772806 0.87526596]\n", - "====================\n", - "18 63.015 200.0 56.433624 [-190.25 -200. -28.875 -183.125 -200. ]\n", - "[0.77782947 0.7740208 0.77715224 0.7852487 0.77923805]\n", - "[0.8703792 0.86372644 0.8684063 0.8587013 0.8679136 ]\n", - "====================\n", - "19 60.641247 200.0 54.19631 [-190.25 -200. -28.875 -183.125 -200. ]\n", - "[0.7771916 0.78043425 0.78965497 0.77828103 0.783827 ]\n", - "[0.8698033 0.8648427 0.86594695 0.8754562 0.8779571 ]\n", - "====================\n", - "20 54.751247 200.0 51.718987 [-190.25 -200. -28.875 -200. -200. ]\n", - "[0.77753127 0.78207856 0.78387165 0.7841341 0.7895221 ]\n", - "[0.87135184 0.87689304 0.8684612 0.8671708 0.8700651 ]\n", - "====================\n" - ] - } - ], - "source": [ - "# META: Initialize the meta strategy state\n", - "inner_es_params = meta_strategy.default_params\n", - "meta_state = meta_strategy.initialize_meta(rng, meta_es_params)\n", - "\n", - "# META: Get altered inner es hyperparams (placeholder for init)\n", - "inner_es_params, meta_state = meta_strategy.ask_meta(\n", - " rng, meta_state, meta_es_params, inner_es_params\n", - ")\n", - "\n", - "# INNER: Initialize the inner batch ES\n", - "state = meta_strategy.initialize(rng, inner_es_params)\n", - "\n", - "for t in range(20):\n", - " rng, rng_eval, rng_iter = jax.random.split(rng, 3)\n", - "\n", - " # META: Get altered inner es hyperparams\n", - " inner_es_params, meta_state = meta_strategy.ask_meta(\n", - " rng, meta_state, meta_es_params, inner_es_params\n", - " )\n", - "\n", - " # INNER: Ask for inner candidate params to evaluate on problem\n", - " x, state = meta_strategy.ask(rng_iter, state, inner_es_params)\n", - "\n", - " # INNER: Update using pseudo fitness\n", - " x_re = reshaper.reshape(x)\n", - " fitness = evaluator.rollout(rng_eval, x_re).mean(axis=1)\n", - " fit_re = fit_shaper.apply(x, fitness)\n", - " state = meta_strategy.tell(x, fit_re, state, inner_es_params)\n", - "\n", - " # META: Update the meta strategy\n", - " meta_state = meta_strategy.tell_meta(\n", - " inner_es_params, fit_re, meta_state, meta_es_params\n", - " )\n", - "\n", - " if t % 1 == 0:\n", - " print(\n", - " t + 1,\n", - " fitness.mean(),\n", - " fitness.max(),\n", - " fitness.std(),\n", - " state.best_fitness, # Best fitness in all subpops\n", - " )\n", - " print(inner_es_params.diff_w)\n", - " print(inner_es_params.cross_over_rate)\n", - " print(20 * \"=\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "mle-toolbox", - "language": "python", - "name": "mle-toolbox" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.7" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/tests/test_fitness_rollout.py b/tests/test_fitness_rollout.py index f4a7438..73d8c5b 100644 --- a/tests/test_fitness_rollout.py +++ b/tests/test_fitness_rollout.py @@ -3,7 +3,7 @@ from evosax import CMA_ES, ARS, ParameterReshaper, NetworkMapper from evosax.problems import ( BBOBFitness, - GymFitness, + GymnaxFitness, VisionFitness, SequenceFitness, ) @@ -25,7 +25,7 @@ def test_classic_rollout(classic_name: str): def test_env_ffw_rollout(env_name: str): rng = jax.random.PRNGKey(0) - evaluator = GymFitness(env_name, num_env_steps=100, num_rollouts=10) + evaluator = GymnaxFitness(env_name, num_env_steps=100, num_rollouts=10) network = NetworkMapper["MLP"]( num_hidden_units=64, num_hidden_layers=2, @@ -40,7 +40,7 @@ def test_env_ffw_rollout(env_name: str): rng=rng, ) reshaper = ParameterReshaper(net_params) - evaluator.set_apply_fn(reshaper.vmap_dict, network.apply) + evaluator.set_apply_fn(network.apply) strategy = ARS(popsize=20, num_dims=reshaper.total_params, elite_ratio=0.5) state = strategy.initialize(rng) @@ -56,7 +56,7 @@ def test_env_ffw_rollout(env_name: str): def test_env_rec_rollout(env_name: str): rng = jax.random.PRNGKey(0) - evaluator = GymFitness(env_name, num_env_steps=100, num_rollouts=10) + evaluator = GymnaxFitness(env_name, num_env_steps=100, num_rollouts=10) network = NetworkMapper["LSTM"]( num_hidden_units=64, num_output_units=evaluator.action_shape, @@ -72,9 +72,7 @@ def test_env_rec_rollout(env_name: str): rng=rng, ) reshaper = ParameterReshaper(net_params) - evaluator.set_apply_fn( - reshaper.vmap_dict, network.apply, network.initialize_carry - ) + evaluator.set_apply_fn(network.apply, network.initialize_carry) strategy = ARS(popsize=20, num_dims=reshaper.total_params, elite_ratio=0.5) state = strategy.initialize(rng) @@ -112,7 +110,7 @@ def test_vision_fitness(): ) reshaper = ParameterReshaper(net_params) - evaluator.set_apply_fn(reshaper.vmap_dict, network.apply) + evaluator.set_apply_fn(network.apply) strategy = ARS(popsize=4, num_dims=reshaper.total_params, elite_ratio=0.5) state = strategy.initialize(rng) @@ -140,11 +138,7 @@ def test_sequence_fitness(): rng=rng, ) param_reshaper = ParameterReshaper(params) - evaluator.set_apply_fn( - param_reshaper.vmap_dict, - network.apply, - network.initialize_carry, - ) + evaluator.set_apply_fn(network.apply, network.initialize_carry) strategy = ARS(4, param_reshaper.total_params) es_state = strategy.initialize(rng) From 518f65b488bb73d7f3b169fe345e871fe448c22f Mon Sep 17 00:00:00 2001 From: RobertTLange Date: Thu, 1 Dec 2022 16:52:17 +0100 Subject: [PATCH 12/13] Update brax notebook & visualizer --- evosax/strategies/des.py | 2 +- evosax/utils/visualizer_2d.py | 13 ++++- examples/01_classic_benchmark.ipynb | 44 ++++++++++++--- examples/07_brax_control.ipynb | 85 +++++++++++++++++++---------- 4 files changed, 104 insertions(+), 40 deletions(-) diff --git a/evosax/strategies/des.py b/evosax/strategies/des.py index 58dd273..91ba64b 100644 --- a/evosax/strategies/des.py +++ b/evosax/strategies/des.py @@ -53,13 +53,13 @@ def __init__( @property def params_strategy(self) -> EvoParams: """Return default parameters of evolution strategy.""" - # Only parents have positive weight - equal weighting! return EvoParams() def initialize_strategy( self, rng: chex.PRNGKey, params: EvoParams ) -> EvoState: """`initialize` the evolution strategy.""" + # Get DES discovered recombination weights. weights = get_des_weights(self.popsize, params.temperature) initialization = jax.random.uniform( rng, diff --git a/evosax/utils/visualizer_2d.py b/evosax/utils/visualizer_2d.py index e9b8c6c..8dcba14 100644 --- a/evosax/utils/visualizer_2d.py +++ b/evosax/utils/visualizer_2d.py @@ -6,7 +6,6 @@ import matplotlib.cm as cm import matplotlib.pyplot as plt import matplotlib.animation as animation -from evosax.problems.bbob import BBOB_fns, get_rotation cmap = cm.colors.LinearSegmentedColormap.from_list( "Custom", [(0, "#2f9599"), (0.45, "#eee"), (1, "#8800ff")], N=256 @@ -23,7 +22,11 @@ def __init__( fn_name: str = "Rastrigin", title: str = "", use_3d: bool = False, + plot_log_fn: bool = False, + seed_id: int = 1, ): + from evosax.problems.bbob import BBOB_fns, get_rotation + self.X = X self.fitness = fitness self.title = title @@ -37,11 +40,12 @@ def __init__( self.fn_name = fn_name self.fn = BBOB_fns[self.fn_name] - rng = jax.random.PRNGKey(0) + rng = jax.random.PRNGKey(seed_id) rng_q, rng_r = jax.random.split(rng) self.R = get_rotation(rng_r, 2) self.Q = get_rotation(rng_q, 2) self.global_minima = [] + self.plot_log_fn = plot_log_fn # Set boundaries for evaluation range of black-box functions self.x1_lower_bound, self.x1_upper_bound = -5, 5 @@ -147,6 +151,8 @@ def plot_contour_2d(self, save: bool = False): x2 = jnp.arange(self.x2_lower_bound, self.x2_upper_bound, 0.01) X, Y = np.meshgrid(x1, x2) contour = self.contour_function(x1, x2) + if self.plot_log_fn: + contour = jnp.log(contour) self.ax.contour(X, Y, contour, levels=30, linewidths=0.5, colors="#999") im = self.ax.contourf(X, Y, contour, levels=30, cmap=cmap, alpha=0.7) self.ax.set_title(f"{self.fn_name} Function", fontsize=15) @@ -166,6 +172,9 @@ def plot_contour_3d(self, save: bool = False): x1 = jnp.arange(self.x1_lower_bound, self.x1_upper_bound, 0.01) x2 = jnp.arange(self.x2_lower_bound, self.x2_upper_bound, 0.01) contour = self.contour_function(x1, x2) + if self.plot_log_fn: + contour = jnp.log(contour) + X, Y = np.meshgrid(x1, x2) self.ax.contour( X, diff --git a/examples/01_classic_benchmark.ipynb b/examples/01_classic_benchmark.ipynb index 1fd1890..b1dae81 100755 --- a/examples/01_classic_benchmark.ipynb +++ b/examples/01_classic_benchmark.ipynb @@ -33,7 +33,36 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWwAAAFgCAYAAACfXUPCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAEAAElEQVR4nOz9d7RkaXbdB/7Od22451/acu3RaJgG0A10dYNGAI1EQuBQhhzQaEmaBXEgUQIocUiJGo5IcZZGEp1GFDHkjChxZkiOSFEERMKIaBqAJECAaFZ1VbuqLm+yMvP5F/a67zvzx7kR+So7fWWlqYq91lvPRdy4EXFj33P3t88+oqosscQSSyzx4MPd7x1YYoklllji1rAk7CWWWGKJhwRLwl5iiSWWeEiwJOwlllhiiYcES8JeYokllnhIsCTsJZZYYomHBO9LwhaRPyYieuLrkoj8lIh82/3etxtBRF4VkT/1MOyDiDwmIn9JRC6ISNne7/8uIlu38VgqIr//NvfvifZ+P3A797ub+9Q+V73G1+95N/bpJvvyh0Tk11/j77f92i5x/xHf7x24jzgG/sX25yeA/xz4vIh8XFUP7ttevQcgIp8Afh7YAf5T4BXgm9qff0BEfo2qvnULm3qyve/t4GJ7v+du8353G38N+HNX/e3F+7Affwj477D34yTu5LVd4j7j/UzYjar+cvvzL4vIq8A/xUj8r923vXoXICIdVZ3do8cS4K8Ah8CTqjps//ULIvJTwLPAjwP/uxtso6OqsxPvzy1DVUvgtu/3LuDinez/vcKDvG9LXB/vS0nkOnim/f7o/A8iErXyyevtZf1XROR3nbyTiHxCRP43ETkQkYmIfE1E/r2rbvPbROQLIlK08st/LSLJif//MRHZE5HvEJFfFpGpiDwtIr/mWjsqIn+03c5YRP6qiKye+N+vby93f7OI/G0RGWMVFiLySRH5++32D9v7nr5q2512/15rn/MrIvJ/u96LJiLnReQ5Efl7ItIFfi3wSeD/eoKsAVDVC8B/C/ygiDxxC/v7tst2MfwJEdkRkaGI/A8i8r9vbzff3jdIInMZR0T+gIi82T73/0lE1k7cpici/52IPN++Pq+IyJ8XkZXrPfc7gYj8m+3+9a/6+9ukJhH5eRH5myLyu0Tkxfb5/qyIPHLV/a77frVFyCbwn52QZX59+79vkERE5PeLyAvtdl4UkT9w1f9v6zhd4u5jSdhX8Fj7/eRl4n+OXcb/P4EfBH4R+Ksi8kMnbvN3AA/8nvY2fw4YzP8pIr8D+FvAP2v//8eBfwe4mgS7wP8b+IvAvwqUwN9qSfAkfgj4DcAPA/8h8FuB//4az+cvYSehHwT+kohsY5fFXeB3Af8+8OswGSht91WA/xX4EeDPA78F+M+Aa+rOLUn+I+Al4AdUdYoRNsBPXus+7d8F+N4b7e917vtjwB8B/gLwrwEz4L++zm2vxu8Avh977f8w8APAf3Hi/10gwt7vfwn4o8D3Af/zLW7/aoiIxCe+ojvYxvcAvx/4j9r9/k7sWFw8ADd+v347Jv39JUwCeRJ46jo7+8PYsfu3gX8Ze95/WkT+46tueqvH6RLvBlT1ffcF/DFgD5OEYuBDwOeBp4Gsvc0GMAH+s6vu+zPA8+3PW4AC33qdxxHgNeB/vOrv/zZGNpsn9keB7ztxm0+2f/sXT/ztVeAA6J/42+8GAvDx9vdf397vz171mP8lcASsnPjb97S3/aH299/c/v6DN3jtXgX+FPBh4HXgfwHSE///C8DhDe6/1j7GH77R/rb/U+D3tz9HmD7956/xfijwRPv7E+3vP3DVPr8ExCf+9t8Al26wnzHwuXZbj11rn27yGulVX2+2//s329/717jPnzrx+89jZLt+4m8/1t63cxvv1x7wx27y2jrgAt94nP54uw/57Ryny6937+v9XGFvAnX79SLwHcC/oqaBAnwLVk1cXWH9deCjbcV6ALwB/AUR+Z0icuqq234Uq9z/xslqC/gHQN4+xhwVb18Y+mr7/W2XwMDnVXV84vefwE4Mn77qdj991e/fDfycnpApVPVXMKKYV7vfBxyo6t/mxvgYVln/E+B3qmp1k9vfCq7e36vxKHAGqwBP4mb7Osc/VNXmxO9fBU7J26Wp39te4o+x4+KftP/66C0+xkn8Few9mX/9ljvYxq+q6uFV+wxwvv1+q+/XzfAIcI5rH+srwLee+NutHqdLvAt4PxP2MfZB+gzw+4AU+GsiMn9NzrbfL191v/nvG6oagN8EXAL+B+CSiPxjEfmO9jbzS9Of4crJoeaK7LLQy4FRuz0ATpBgftXj75z8RU2GGJ/Y36v3c46z1/jb/HYb7c+bWBV7M3y23d5/fxUJglVqazfQfh8/cbsb7e/VONN+373q71f/fj0cXfV7hZ3oMgAR+e3A/wdbeP7XsePit7e3vfo9uBVcVtUvnPh69g62ca19Prk/t/p+3Qw3PdZP/O1Wj9Ml3gW8310iX2h//hURmWEf2H8dqyzmH4RTwP6J+80X6Q4AVPU54F9tK7VfA/xXwE+3i0Nze+C/g8ktV+NObFVvq+Jb7bDPN35wr87NvXj1fVucBv55+/M+30j818L/iFVePykiv0FV/9mJ//2j9vsPYlXm1fjBdt/+8U3292pcar9vX/X3q3+/U/zrwK+o6r87/4OI/Lq7tO2TKNrv6VV/X7+Dbd3q+3UznDzWT+Jtx/oS9x/v5wr7avwV4CvYghTAl4Ep9kE+id8BfF1V31bZqWqtqv8A+DPYh2gNeB6rJJ+4qtqaf+1z+/iNVzkMfjtGdl+4zu3n+BXgN4vIyQXRT2Oa7/zS/+8DG3JrTSf/R+CngJ8VkZOXzP8I+CLwR6/hhDgL/Cjwv6rqa7fwGCfxBkbav+2qv//gbW7neuhgC2gn8bvv0rZP4s32+8fnfxCR78FOgLeLW3m/Km5e/b4JvMW1j/Uh8KU72Lcl3gW8nyvst0FVVUT+C8wF8v2q+vdF5L8B/s8i0mCE+K9gWuQPAYh1Rv4prCJ/GauS/jDwjLbNNyLyHwH/31Yi+FnsA/RBzIf8r7WSxu1ghlXwfxI7MfxJ4CdU9as3vht/BnMT/F0R+a+wqvy/xD6M/0t7m88DfxeThv5zzFFwFvi1qvr7rnq9goj8G+19f06sGebF9nX8vcA/xPzt/zWmk88bZ46Bt9kebwWq6tvn/CdFZBdz7PwgV/TVcN073xo+D/x5EflPsZPbb8FcJXcb/ww7if+3IvJHMbnhD2HEeLu4lffrOeC3isj/hklnz6vq6ORG2vfyjwF/UUT22+3+Oux4+SOqWrDEA4ElYb8dfx1bCf9DWPXyfwEa7MA9jS1O/h5V/Z/a21/CdL7/FFu0OcKIal6lo6p/XUSGmB3t38YsgC9j1emdLNb9T8AIs2r1sUW3H7nZnVR1V0T+BeBPA/+/9rF/BvgDcx2yJdvfDvwJzJGwjVVe12wkUtWmtS3+HeDvi8j3quobqvplEfkuzGL2X2Ja/kXM0vcnVHXvDp43wJ/FCO7fxSyNfxuz5v04d0Z4J/EXsRPpj2IV6ecx++NdbTBR1ap9jX8c+JvYVdiPAH/1DrZ1K+/X/wmz/P00toj+L/CNXY+o6v9LRHLs+f8oVnX/R6r6Z293v5Z49yCqyxFhSzy8EJH/HviNqvr4TW+8xBIPOZYV9hIPDUTkW4DfCfwSJoH8S8C/xYkrmiWWeC9jWWEv8dBARD6A2Sc/CfSwpqS/CPxpXR7IS7wPsCTsJZZYYomHBEtb3xJLLLHEQ4KbadjL8nuJJZZ4P0Pu9w6cxLLCXmKJJZZ4SLAk7CWWWGKJhwRLwl5iiSWWeEiwJOwlllhiiYcES8JeYokllnhIsCTsJZZYYomHBEvCXmKJJZZ4SLAk7CWWWGKJhwRLwl5iiSWWeEiwJOwlllhiiYcES8JeYokllnhIsCTsJZZYYomHBEvCXmKJJZZ4SLAk7CWWWGKJhwRLwl5iiSWWeEiwJOwlllhiiYcES8JeYokllnhIsCTsJZZYYomHBEvCXmKJJZZ4SLAk7CWWWGKJhwRLwl5iiSWWeEiwJOwlllhiiYcES8JeYokllnhIsCTsJa6LEAIhhPu9G0sssUSL+H7vwBIPHlQV7z1VVeG9xzlHFEWLL+eW5/kllrgfEFW90f9v+M8l3ntQVeq6RkRQVaIoAsB7v/gSkSWBL/F+gdzvHTiJJWEvscCckOck3DTNNxCyqhJCWBL4Eu8XLAl7iQcLcwkkhEAcxwvCrev6pgR8MwKP4xiRB+qYX2KJ28EDdfAuCft9jhACTdMgIt9ArrdC2FfjJIE3TUMIYUHgcRwTRdGSwJd4mPBAHaxLwn6fYk6sTdMsiPRq3Alh3+hx5lX81YuYSwJf4gHGA3VwLl0i70OoKk3ToKqkafquEuZJeWT+2HPpZO5CGY1GbG5uLgl8iSVugiVhv88wr3adc7ekL9/kCuy2MZde4tgOPe89Tz31FJubm1RVtazAl1jiBlgS9vsE11tYvN8QEUSELMuAb6zAlwS+xBJXsCTs9wHmEghAkiQPNOFdXYHfiMDnJ54H+fksscTdxJKw3+OYk92c5B5EcruR7HIjAi+KAlVdyDvzBdIH8TkuscTdwJKw36NQVcqy5MKFC3zgAx94YCSQq3G75HqSwLMsWxB40zTUdb0k8CXe01gS9nsQ84XFpmm4fPkyH/rQh257G6rKyy+/jPee7e1tNjY2rmn9u9+4VgU+txDOCfzqLswlgS/xsGJJ2O8hXO2tTpLkjlweRVHw1FNPsbKywvr6OhcvXuQrX/kKSZKwtbXF9vY2a2trd61qv5tOFBEhSRKSJAF4WxfmksCXeNixJOz3CE56q5MkwTlHCOG2yXBnZ4evfOUrfOITn2B9fR3nHOfPnweMyPf29nj99dd59tlnybKM7e1ttra2WF1dvSPie7fJ0jmHc+6aBP61r32ND33oQ0sCX+KhwZKw3wO4nrfaOXfLhB1C4LnnnuP4+Jgnn3ySPM8XzpI58jznkUce4ZFHHgFgOp2yt7fHyy+/zPHxMd1ud0Hgg8HggSS+kwR+eHhIt9u9bgU+fy0fxOexxPsTS8J+iHHSW32ym3AOEbmlAQTT6ZSnnnqKU6dO8ZnPfOaWCarb7fLYY4/x2GOPoapMJhP29vb4+te/zmg0YjAYsLW1xdbWFr1e74EkvhtV4NPpFOAbclAexOexxPsDS8J+SHEr3up5pvWNcPHiRZ577jm+7du+jc3NzTveHxGh3+/T7/d54oknUFVGoxG7u7t85StfYTqdsrq6utDAO53OHT/Wu4mTBK6qb3OhlGUJsCDvZZTsEvcaS8J+CHFSArmRt/pGlaD3nq9+9atMp1M+97nPkabpXd1HEWFlZYWVlRU+9KEPoaocHx+zu7vLF7/4RcqyZG1tje3t7Qd2DNm8mr6awOcOnLIsl1ngS9xTLAn7IcLdai+fTCY89dRTnDt3jm/5lm+5J5f4IsLa2hpra2t85CMfIYTA0dERu7u7TKdTfuEXfoH19XW2t7fZ3Ny86yeQu4E5gc/37eoo2SWBL/FuY0nYDwlOju56J+3lb775Ji+++CLf/u3fzvr6+i097rsB5xwbGxtsbGxw8eJFvvd7v5eDgwP29vZ48cUXUVU2NzfZ2tpic3Nz4bN+kHCtJMIlgS/xbuLB+xQs8Q2YE8A7yc7w3vOlL32Jpmn43Oc+t1hke1AQRRHb29tsb28DlsU9J/Dnn38eEVksYD7ITTy3Q+DLaTxL3C6WhP0A41re6jvBcDjk6aef5vHHH+fxxx9/KEgiSRJOnz7N6dOnAaiqiv39/Xe9iedu4noEfnR0xBtvvMGHP/zh5TSeJW4LS8J+QHG7udXXgqpSVRVPP/003/Ed38HKysq7sKf3BmmacvbsWc6ePQtcu4lnTuB32sTzbuNqAu/1eosKfJ5EOG+1X0bJLnEtLAn7AcNcqw4hkCTJHV/613XNs88+S9M0fN/3fd9d0YAPXwnM9pSzn7r/3YAPcxPPnJhPEniapm+LFlhmgS9xLSwJ+wHCXAJ57rnnOHv27ELPvV0cHx/z9NNP86EPfYjRaPSOybqeKm/+sufghUDwwmRH+eBvjIjSB4c8HqYmHlW9rm/+RuPUlgS+xJKwHxBc7a2+E3eGqvLKK6/w5ptv8qlPfYp+v89LL730jvZr//nAxV8N1GMh6QouVY5eg+d+suHDvyUi6z+Y2vGtNPFUVcVsNrvnTTzzCNib4XaGOSwJ/P2BJWHfZ1zLWx1F0W03k9R1zdNPP02WZXzuc597xy6Kcqi8+vOe49cg6TqSrlJPBZcI3W0lNPDCTzV88DfGdDcfPNI+idtp4tnc3CTP83d1f+aSyO1iSeBLLAn7PuJ63upbzQCZ4+DggGeffZaPfvSjnDt37h3v0+VnA2/+UkAiIekrzQQkhnwjECWOcgz5KhTH8OLPNnzg+2MGZx9s0j6JeRNPlmU8+eSTb2viefXVV6nrmo2NjXetied6ksjt4lYJfDnM4b2DJWHfJ8w/WNf6IM2jUW8GVeXFF1/k0qVLfPrTn6bX6133drfyQZ3uB175e4HiSIlzKEcgFXTWQSJoSiHOQQPMDqGzLhTHypu/3HD+uyNWzj943uhbwckmno997GN479/VJp5blURuF9cj8OU0nvcOloR9j3Er7eW3QthlWfL0008zGAz43Oc+d10CmAdA3eiDGbxy8QuBy88GJAJfgq+UfB2aEiRSXCKEcTCiXoPZsVJNlP45KI6UN35JeeRJWH3k4STtk3i3m3juVoV9M1xvGs8v/dIv8alPfWo5zOEhxJKw7yHmC4s3ay+/WY713t4eX/rSl/jmb/7mRWPJ9XCzbY0vBV77hUA1VuopIEpnQ5gdgIshy5RqBATIVh3VxEi7f1ZoCqU4gmzFUQ4Db/7TBvmssHL+4ZFHbgV3u4nnTjXsd4r5caeqb/OAL6fxPDxYEvY9wNWju25WkV2vwlZVnn/+efb39/nMZz5zS+6G60WshsasenvPKfUIcEq+IRQHgq9hcA5mhwENQr4q1DOlPIbOliCRUh4rSV/QoJRDJR84ilFg50s1URrT2374K+3r4Z028bxbksit4GR1f6Ms8CWBP5hYEva7jJPt5Wma3tIBf61Fx9lsxlNPPcXm5iZPPvnkLX/gr7Wt0YXAa//IM75ocodV1EJ5qAwegeJAme0LSU9oKqUcCmlfSLeVcghxJrhEqcdK0hVolKZWBmeFYhR466maRz8j5KvvrUr7erjdJp57JYlcC/OFyGvhdgh8OY3n/mBJ2O8i7rS9/OoKez5n8Vu+5Vtuu5nmpCTiK6uqL35BkRMV9XQfBueUplSmu0LaE4KHagxRBp1Tii+hnkCSQz1TokyIMqUpA91tR1N5ipGQ9R3VWLnwqzWPfS4l6bz/PtA3a+JxztHr9RiPx/e8iedGhH01ltN4HjwsCftdwDvNrZ4TdgiBr33tawyHQz772c+SZdlt78tcEjl6zfPyzwWaAvI1mB0Ks33onbIqe3YASV+IM6gmEKWQrSsSQT2FpCtUY0VD+/M00NsWEKEaB+KuQ0OgHEPWt9te+ELFY08muPjBq7TfrdjYq3GtJp7nnnuO8Xh8XybxeO/vWI5ZTuO5/1gS9l3GrYzuuhmcc1RVxS/90i9x+vTp25qz+A1oHK/9fRi+Eohz8BXMKuhugkqgKRwutWq5HoNLIB0oce6op0LcVbSBaqxkA6EcWXXePytUI3OPuBSaqZL0HPXMSLu7DirK/ssN2x998IYRwLs/sf16j5nnOd1ul8cff/y6TTxzF8rdbuK5W/r5chrP/cGSsO8i5peLNxvddTMcHR1x4cIFvvu7v5uNjY073p/DlwPDXzyH9JRQgC+guwXVVJFIIDhwSj2xDsZ0RREBiQVEOT4cUV2uyNaUJPRRSRiciSiHSjWEpCfUUyVKW017ouSbAgJBrTof73rylYbBmQfrULtXFfa1MB+aDDeexPPaa6/d9Saed1Jh3wjLaTz3Bg/Wp+ghxVynjKKILMvu+ED03vOVr3yF4+Njzp49e8dk3RTK6/84sPdcwA9zahUGp4XJXkCckA2EZqb4SnCpkA6gmVmzDBFM9z3Hx8f0NiP6YYPGe1gdMRpNGL6kZKtCqj1UU5KuoymUbM1sgKFW0r5Qz0BUEAkcvFKTrzqSzoP1Ab1feuuNFh1vp4lnY2PjtgdR3I6G/U5wvSzw4XDIq6++ykc+8pElgd8BloT9DjFfWHzxxRd59NFH71iDHI/HPPXUUzzyyCOcO3eOt9566462c/hS4JV/4CkOjUCjQYUGpZ4qg3NCOVRCI0SpkK5ANYRoBdI+zA6sIWfs91jpb5OEhM52QIhoijXWHxOK40Bd19TRlMn+BIY1va0EnfXorqQoSjWBpCM0pZJ0HXWh7L5Qc/Zbb80lcy9wPyvs25El7nYTz70i7KsxJ+d5JX7SB76cxnPrWBL2HeJa3uo7nf49n7P4yU9+krW1NQ4PD2+bUJpCefUfeHa+rMSZLSwWR9AMUzof8UgZMdsXokxI++YAcZHSP29EzViZcUQxU9ZXztLbcrgkUI2tMQYJlMdKvhLhRkInTuk+IpRTT92UFOWU0ZsjJPN0kh5ZSEnTmGoK+ZoQJYHpYU1v48HRsx/ECvtmeKdNPPeLsOeYxzHczji15TSeK1gS9h3gWqO77oSwm6bhS1/6Et77t81ZvNUskTkOXgi89Hft9nMybkronYGpayj2Hd0VIemZNc9Xgf45RzmE2R7EK4Gd1w/J0ozTj/SIO4ovFZw1yVQjJR20C4qjQP+0dTlWYyXtRbiiQ7ffITQB3wQaVzAZTTiMRvRWOmiVk8cZk32ls5Lg4vv/wbvfFfbdIp/bbeJ5EAj7Wo9/IwJfTuO5giVh3yau562+XcKez1l84okneOyxx9528N1qWl89U175e4HRhUBooClAHHS2zKpXHJmO7PJgmrJTutugQZjtW1bIcLdk75URG2f65N2UOBHUA2J2vigXRKEaKZ1tswhWYyUdCNVEqWdW0fsSkm6EOKG70ic6Ewh+BY085bTk8PAAwbE3bFg9YyRyJzbF24Gqor4lSCe46O0f8PvZvPJuPfbNmnjmZDccDu/LJJ55hX0zLKfxXBtLwr5FnPRWn6wE5ridhL3XX3+dV1999bpzFm9lW/tfD1z45cDkkk2BiRLobAihMZISEaJECccxIYLuViBOHdXIFgiZKntvTGhkwvaZLTobEaFWQrAmGWuMCTQz6G5AlCrVDGuqIVCNWJB2aKzBJsqUdEVpKgUERXEa0x3EdH2XKBXKsmI83OXVV79A0zRsbm5S1/VCWroTzCv/4kippoHgoThUgoe0Y1ZEMP+4L8ElQpR75NI2u19rSAe2EJsNBHHv/of+XramX93E89prr3Hp0iW+/vWvMxwOWVlZuaeTeG6VsK/GchqPYUnYt4Bb8VY75/De33A7dV3zzDPPEEURn/vc565LUDcKbGpm5gB561e9XSJ2hSQxn3SUGmlWQ6jbhcWo1xB3U3NsOHsu0z1lWO4TRylnzp1euDuawnzYccceJ8qhd8q82lahBqoJpD1HPTXSTgYQZ+DigG8gFNa6Pl9w9JUSOeucBCFJUs6sPMbHPv4RmqZhf3+fN954g1/8xV982wLbzUKUfK1MDhpGb3nqIiA4Qq24BNQ7c7w40EZJ+4DYVYCq0lSBugCGK+w+35D1LbxKnJCvQWfNka87etvOWu/vMu5Xa/rcdrexscFHP/rR607imRN4t9u96/twp4R9NW41C3x3dxeAxx9//B0/5oOAJWHfBFeP7rqRHetGVfHR0RFf/OIX+fCHP7y4XL0erretgxcDL/2MJevla1YtNlOI1qB/XpjuKqiQdCHqgJ+1JJ6bS6SeAWnFweExq2sD1k7nBG+uDhe1vuqJosGkk7qEeiYLAo5zB2qkna+YJTB4DyI0pVWwTakLSaVpH78plDgTfKXEXWjaD1Ycx5w+fZpOp8Ov+3W/jrIsF/rrM888Q6fTWRD4/PI9NIHRTsPkwKONgFOixKyFLhGaGWhQkhzKY3vd4tzCq1QVFylOBO8VSWs663ZVYtV3oJk59vc94IkSm7bT2RAGZxy9U+6u6O/3K61v/tjzE+GNJvE888wzFEXB+vr6XW3iuVuEfTWuR+A/+7M/S1EU/NiP/dhdf8z7gSVhXwe3214eRdGiCr96Oy+//DIXLlxYzFm8Ga4m7KZQXv67gZ0vB7pbgsyMjKJEGTxqRF3P7JI+NDbKK9+EdADj12KqRhhsC8e7Y2b7Jec+tkFERHlsgwmynlBOrPGlswmqQjU1cq0mEOq2ai6UtOeIemrVbAy+ETQI4uykkHQsMGpO3oiQdJSkG3AdJUqEOjRUtaNz1Qc3yzLOnz/P+fPnAZhMJuzu7vL8888zGo3YWN1m4E6Tpx1oYprS4yKrrKMUUNqKWNHG/OWIgreTkYYA6qgnJpeoKLNDu5IRgbjrwEO+alp9qKA4DhTHMHorIAKdDcfgvGNw1hEld0a69zOt70aNM7faxDMn8Dtp4vHe3/UJPtfCnMCLorilz9zDgiVhXwPXG911I1xLEqmqiqeffppOp3NbcxZPLjoevBB46X8LRCkoynTPJItsW/GFY3JZ6axbTnU5EuIuDB61EKd6IsT9gFaBy68dka55Hnl8g+pYCJERejVq41E3TA6pp7ag2LRhT3FHCYXaYuamEnxboQbLz45zI+i042iqgK+N/FDobIJEnjiXtroVQlAiF1FWFXmW3fC17fV69Ho9nnjiCWbDmoOLUyZHJZf39wjeE0cZufTJOilaRITGFkDrib2HcWpuGfVqQxhiRzNr9eyBIhMhXxGCt0XJudYdJYAIvm0CcomiNVQTZXTRM9337HxV6G461h6L6J26vdCjBzWt72rcqInnpZdeuqMmnpNdnvcCo9HoppnxDxOWhH0V5j7QeVV9pwl7BwcHPPPMM3zsYx+77TmLzjl8Kbzw057DV8Li0j5fNXLJBo7i0AixmSmzQ6uA+2eszXx6GfINKIZKdeSoOGTrkRViv0Y1gmzFGmjKEaSrEKdWDUctsZZDJVuRdqiBMtiGECAEY+J6qiR9Iz9fm5xSzYzc4lSIOooScJGgAqEBnKJeIAKH4FWpm4b0Fj7k5bRmdlwTS8LKakon6iMxzEYVRT3lePcA1JF3UrLQI+tkuFgIjZDGWJWtQqhtMdXFgWII0iRUEyXu2HONUyFu+57qqZF38OBru3KJO5a5ErxJR8MLntFbns6Go3/asfZ4RJzd/Hi534R9u92Rc9yNJp6mae4pYU8mk2WF/V7EtbzVt4O5rW8+Z/Hy5ct8z/d8zx0t3By/AsOffxSX2QmguyUUh6Z7RpnYYprCdE9J+w6JPVHqmB20JH6gFAdQMaGMDlkbbJBon2RggU3FsdJZNxJ2YrJI8DaUIFttyXyodLbMAthUlt7XVEqSCz4Ei1rtzmUSQdKARJYf4hs7CfhGiTNH8IE4jgg+ELVVtohQVdVNCbupGqbHFb52oEAQkq7tY5ZmpHFOzynEgdmwZjqpOGyOkRCRJjlZltHpJqbfY/taTe3KgaKy4QyFbVpRmlLMgw5kA3utQ2knnRAUqUyLT3vmUUftvSkOA/tf96x/0KrutH/94+dB0bDfKW61iWdra4v19fVFUXMvCXs8Hi8J+72GO82tPol5wt4v//Ivs7Kywmc/+9nb/mD4Snn55zyXnoIwSeieF6Z7SnEQ6J0VQgXlEOqxkq9b8FLcUZrCHCBgkanpQNnfPyTuKiuddeJMCA1Gwut2aS+xkET2t6aw8V/l2LoZOxtisaqzQNIzd0WoIYqhbnXsplBcZGFS3gcj6BokEdQHQmibHxpBRQmNkTmhXSh0EU27RnCj92UyrGgK+71ppZlQm/RDY+4TFwu+dHT7Of3VnOBXaBpPURZMi0OOdmriOCXvpPhJD6cRvgHBURy3EknHTjJqzA1qyYahtv+nfUx/n9jv9czSC5vKrkbqNovl8BXP4auewVnH1kfiaxL3/a6w3y39/HpNPG+88QZf+tKXyLKMqqpYW1tjY2PjnrwGywr7PYST3VTX8lbfDobDIW+99Rbf+Z3feUea2dErygt/xxM8ZCug+47pDvQfAa2EqbmT6G4JswPLtc7WgAC+NMdIvg7To5qdg10GgxUGKz2OdiY0U8fgLMwOLaWvswHlKABCNnDUE5Nd8nXBpUrTyhtgXZP5AOoCosisfRIJ3W3wdUDVZJKmAJeCb6UV05Otql58jwXfBFyqEN5ul7wWimmNL41EtYK4Y5W1+kCkEfXUCFwD+EZBAhoifAXg6Pf7ZNJDOwpxw/SoYlSOaEJl1sXgcZlHvUPELTRsl0CUmxbvxWQo1B4bbAEzSpXiOIDYgm3aBxFb/PQVjC55ypHS3XBsfjgmzq+Q0/1cdLyXnY7XauL5whe+wFtvvcWLL75It9tddGG+W0088yER7xW8bwn7agnkTg+W+ZzFy5cvs729fdtk7SvrVjx+1VwavhZEwK3M6J9TprsW4mRjvJTJXmBwTgiVUh4ZWXY2hNmhMjycMvWHbG6cQuqEagJx3+MnDl/B4JyFN6FyotXcCCjOzDnhEMAyRJJ2EEE1s0kzUSZEueArT1OYPONrJckjmjogas0y2syra7XfvUWuarDYViFC1Xzk9XUIuywqynFDqE17Do2aHJI7NLQVfw7azKt/AXFtY4xZEENjzT0i4JuEwVpCXXTI+o7xQUlTj9jfPSRQk+U5nW5O7FKci2wgMZb9Pa+SQ22atotYjEqLUtP9mxlWmWOLueLsSub4gmd0ybP5kZi1R60T9L0iidwuut0ueZ7ziU98gm63+7ZJPO9WE89kMmEwGNyFvX8w8L4k7BACBwcH7Ozs8PGPf/yOD4z5nMWtrS0++clP8tJLL93W/Y9fU177hcDRKwHBCLCzaRWjXIpbq57NUZzuKf0zgMBs3wgh34DiQJjsB8p0D184NjpnEe9Miz5WmjIi264JmjE7ENK+o6ksHyTuOqIs4FzrctATjTEza4yJ+5CkgsQBxBpO0m5EU4YrLexFIErlisbdKHEegSguF4IEvPOU2uA1QAWRE9LIwTUIO4TAaDTDNQnaCC5StBaaWokTO4kEPEkSEQJIYrqo2QghzqyjE1oPdmGvKVhTTGigsxYzqwJbp05RT5WqrpiNCspmiAB5LyONOnSyjGrcWv8cdDZl4TTxlZ1MVSHtC9XEXhMXOerCNPBqEkDg+I2G8Y5n+2PJe7bCVrWmrWpssl05kja7xha4mylcenEL+dUELZTQdHHpYxT7j+EaGHcq3ni9ophNyT76Oo//ywXb2++siWdJ2A8xrm4vnyeC3QkuX77MV7/6Vb71W7+Vra0txuPxLWeJ+Fp54x8FXv8nCmppdpYFYs0fqKLe/MIi0N22S/9yaHpttgrlsenVyUbN3u4unWKVldUuSdduVxwpvTNQ7yvNMGJwytGUlgMSd606jXJzPASvhBKSjkPrOWnbJBklEKVKU0KUOSBQTQNJxxbfko7D1wHUKtsohzRXXBqoQ4M4x6yucc4xrSpEhElZU3pP5BzTqqa56j04Go5pppCoSR2Wqw1Jbm4Wlyixi6mnVsHHSUTdkmjadabtt1cE6tuW9NY3Xo3a9yCARFa9ZwNHJhkaMuqJBSRVdcUsTDjeOcA5Ryp9+hspxTBGsKHEOMUXSgi02xeydfNui7RrDJmQdKAcK75W3nqqROr7RyDvlLB9qUz2lNmuHWOzfZju0oaBCcWhWSajyI4Pl9oVSahscbvYSZg2DhcDCqF1QLkM/DRjbSvjke8d8OEf2L4rTTyTyWQpiTyMuNpbHcfxTVvJr4X5nMXRaPS2OYu30poOcPxG4Os/qcwOlN62Vc7FkdDZMC/05LJpy4LJIE2laBA0WNt5M7WKpbMJs2rK7utT1rc2yXsp9cTIurNplWZ5ZHps8Eo5bkksVmsi8a1lrbSJMVFiQU5JxyG5NVfEKZRTW2iMUnNPJF1HXQarLp1JFdmqWeXi3HJGXOIAJYli6uBJoog6BBzgbHWUbpIwriq8BuoTawdVVTMZ1vSiDk0huMgsdi5quzCBJHM0lX3InXOE1sUikWnz85NwPda2JZ7WHaIkPauU6wY4jtFgi6txx05oSR/A0XE51SSHTmv1jKYMR0fUEyWOUvI6Ju90SPMYpfWv94RybCczoX3dMnvctGeOmhCUqO6z/3XlzMfNSngvcTsuDV8p44vKZFc5fhWmOwEiaKZijUrYQrjEbUJkYS6l2Z5Q1vZzsWsOm2wd/EioDxLqriOKMBnJ2dVJ0V41fvx3wiOfs9fkbjTxzDtq3yt47zyTG2CeMTCfajEPkrldwp5MJjz11FOcOXOG7/me73lbdX6z1vTgldf+oXLpqUCUAQqTHdM78zVlctkO+nTFQVDCvsMlSpraZWZozLmRb4B6ONg/pvEV22c2CbWjnkC21rZnT2ltZwE/iZC4IUot80NiI+umtIowymRB2ulKIIpBImuCqWoh7tiHUr1V+fVUibtClAhRVxFp/dYu4BuHxEbikmh72oHYOWrvyZPEqm0RnAhBlU6SMHRXCOTwaIw2QmhDmHxlsoavwKWWuDeXKOLsys9JRyjHVkXHmXVY2tWKLQbOFwjrqWnSzQSIbbHQxUIxNG2/nirZinnMs0HbPJPE1JMBeXeA9BWNa2bTkuPDId57kiShuxFRlx3AJCdU6Wy6xf7VM3PJ5KuCjCOqkXLx2ZJTH0vJBvdOHrlRha2qjC8pwzcCwwvKbM+umkJtBYMvoTi24czNzCyPLlZ8aVp+0oXqGJIuJD2YXDSy7m4L44t23ErqqY+hKM15A8LxK5AN4Lv+AGx+7PYn8Vyvied+Rui+W3hPE/aN2stvl7Dfeustvv71r/Nt3/Zt1xzddaN41fFF5eXPB0ZvBnxpul5nCyAQfNutuCWUh0o1VNIBxJszZgeWehfnkHWtgy9Iw+7hLpn0WO1sQ2gbapo2QtRZlkY5hLgruErxQc15Eq5Uq0nHrGioLAKcNFjGhlYmLVSzYATWs+ozHZg7QmJF4oA2Fr8avJrPOgQcDtVAhAVYReLwGhDaCe5AJ4mZ1Q2okjhH074vx6Mxs1lD4jOITGYwB4ozXTyFUAlJDjiTM5KOSRPW6GKvQWjMYgjWIm9T4KVtmqHVoBUJzlweoyuviURQDG1NoZq0FbdaZV5NFCcQRymuk7C2YbfxUUk5rpnUh3ipyKRHbyOlHGdEicO51gbYF+oyEOLKOikr5eC1irVHEzqr98abfDVhqyqTy8rRq57hBVtYrSeWO5MNzC4pETQT6xrN12F8ybpBO+tKPbQK2yU2MzTKIcKcS9m6EX4ztUIj6cLB1yDZgO4puxokwObH4JM/Ar3tW7/auFkTzz/9p/+Ul19+mSiKKIrijiZBvfXWW5w/f/4p4JuBvqo2IvJngU8BT6nqj972Rt8h3rOEPfdWX6+9/Fbzq733fPnLX6YsSz772c9e99LrWpJI8MqFX1Ze+bxHg1Wl3S2YHlglXI2EbMUWY2Z75qnubgqzPdBhh+wDbTbITEi6iqzMuPTahJXBJp1BupAxLC4UimOrapOea9P31FqwawuMkqiVBibWVp70zf3hKyN5X5nui2p7Ge8oJ94W2zba4J7U4RuI1BFoW9GddTwuquvYKiuJIXIO7wNJHFM1DYLJIkGVrG2aCSJMi4LRsKAqlX5mFa6LQRuHuLYbc6qoBltUPFFl11NAjURcm+8RZVhCX6ttazAJZ65huxTolmi48n+cXeXEmXVsBh9oClm4P5KuuWnKoVXr1bRtXY9yYlL69AmqaFowmxRUswlOhCRN6W3GVEWKOCHShLQHhHYB/NWK9ccSuuvv/sdxTti+Ug5f8Ry8HGimUI9tAdVXlg3jxF6rKFP8TJDYIhHKI3sNXCQM37B1kKRjlbXEEOcmdYiYHDTvEZj/3SWBZNCeeDvQPw/f/n9gYSO9U1zdxPNN3/RN/PRP/zQ/93M/x6/9tb+WlZUVfvAHf5Af/dFb59i2MPt+4CcAROQ7MeL+NSLy/xCRT6vqr76jHb9NvOcI+1qju66FW6mw53MWH330UZ544okbLlBeLYlMdgLP/4QyutAm6zX2oWhq6J2mHYJrGna+LiABEcdsR+lsCuwr5bF9SAaPKLuvTKjritPnNxB1lo7Xgd4paStxazlvSmtP72wKEgujXfNAp2uWN1K1nuEoMzeHYlo2dVslle08xsq09M6WWksktvDXFG0CX60kmcM3gShxBG/VdAhWXdt3WRBi1L52eRxTtq97FkdMqxqA3cMRoVKiEIM6EMCbW8VFCurMyyy2v1dCntpFLa9EsVssPsa5acaqVl3HHdrb2kJrlClulONbe2CSn2hHryFEdt+sB1VhMa3WOGMJfmCX+KGx9zHpONRDmgq+6ZCudEhOCdPjhuAKpqOCoj4gIccHT1U29NZj03FRDl+39ZXO2rtbaftKOXgeJm9ZQ1KoLOEwyswSKWKDL0KjdDeEZmJXHagtjGdrJtH50k7ixZEwGUK2asf3dMdsogKM3jIS76zB+II9fkAZviYQ4NFfC9/2b/OuTCA6deoUP/RDP8SP//iP86u/+qtcvnyZF1544ba2kec5qnp44nP/GeDz7c9/D3gSWBL2neKktzpNbzzwVURuqHG98cYbvPTSS4s5izfD/LFUlTd/UXn1HwY661Z9FkeCxMrKYzDZsTQ9cdYEM90PRJlQjy1drgKm+4pLAp1TgXoKF54/prMOg2QdPxWiXOmfsxX6emaSSDUOlEMhHQjplpG9OIi6gfoYypEtDtqan1XbWkDt22S9mRKJEZrEQndgpO1Lc2ec9FtrANSqaiIjARcbibsYvFdw2nY1QuwimuBNFnGO0DQkkckRTQikIjR1oK6FPDLPuItkEeYUakdorsSnEiz32nzRJm/EuaBNq907I3WxiwWilLYpxt7vfBWKAhA7cUSpLRYKRjZgHY7BQ9UoiJKvO5qW0H1pj5/02jhX7DWIO+asaWq70qlnSncjQn2XTgMrrKLiuXxph2l1xORi6/pxXdIsY3i5IspT0vzuk7aqcvhyTf3SNod9QYJdvfh2X5upgqMN9AI/c4wvK2lfobHXRCIlVLbgGHeNoJNuu9C4by9v95RQHCq+ME1bBMYX7aTa2RJ2nu2iW/Dx3wEf+W3v7oLrybb0k9X3O8Aa8HL78zHwiXe6wdvFe4aw70Z7OVg4zbPPPouqvm3O4q3AjxO+/FcCk0tWdY0vG4GkA2uIGV9Skp7DxUao9UxtPmJpt7exXUKoA1EIjHZrJn6XXnebjAwXQbZh+uJsF/INoWzjP+MO5ANr4KiGV6QPX0ZIpyDu2CWvS+xS11e2qFZNddFWHWdGXsG3Af+JZ3QwIa1isjyDdjiBr1s7X6MkmV0uu1RpCNQSKH1DUKWeeXDQiRPSyBFHEU17FZJEMWXrwT67ukZVBkQjEhfZ4uYkoKhlkEfWFk8QkgxCq19IAIfiEte2jCsq1hWpDSCQt9p90hHqaSDtO7PYNXaSzdcg1MGslG1Cn4va4Q9iVXfWv2KxTPtCPQtEqVXz4iBpXxNQ6kJby2ObWojZB53YSUA0Iu4Htra2CF6p64aymXE0nBI1ntHXU7qbwvaprTsOaboaw52ao68HpodQHkfQEVsgxgY1R6llvvjWbVMemuMjX3GMLtrJqbthlbWLzarXzKzyjlNoKntdJFZ8YQWE2zZNWxUGg/nVCCSnZnzq34dz331v2tLvsqXvGJiPiFoBju7mxm8FDz1h325u9Y1wozmLN8NbXwgcf/4JohVFgN42zI6VdEUoDqwhxiJLrWJbeVQoDpTpviwaMqb71vYtCr7xjMsxa2tn6axHzKtIayGHooLiwBYCJQrm7JhalVdPtB0FJtSHVvUmA6WeAPOGjpFSjuz+Lm4XEp3JCHHumE4KRntD+msdQgOHR/tGqFlEZyUjTTO7goggSLAqs2lQEeq6IY4iCt8gKhzMCoq6YbWTs55nRGLyyjQEUoQ4xEx8w4pPiLqA2kKr+iukiAq+tkxqcW4R0JR0TDZJOrJoYvEVNE0gyRzVxBwbYHkpqF0x1CMg9u3/re3dxW2be2ljz4CFnGXHmlkAsxWr+OFKW3w2sMRCsJAsF18JxwJMu06FOBeyMrfJOAgJCUmasHkOfBmo6prpwYRffvmXAdjc3GR7e/u66Xc3QjluuPzlmuke4KO2MAjm+vBQDO0Erd6KB9da9lwKaQzFgRFynAmjN9ppRF2YXLaf83WY7rM4MRZHJoVEHahHbQNY3AaItfr16d9ymXPf/ehtPY87xWg0utuE/U+B3wf8DeA3AH/5bm78VvBQE/atjO661e289tprvPbaa9eds3g9lMPA839LOXjBHBedDZgdtBXKGqB24E53TVftnAqExjG+aORqTTAWwjQ4J5Qjz/7lY0KdcvZDq/ixEGdWxQhGGrMDyFaFZubNP1wY2YTGRoVZ4p6NvMrWAuODiHo4n8Fo7ejZwDRgl6qFOpVWgUsER/sjCj9mc2PbqqjcsZr1TF7xJaWfMpxOaEYVg9U+SZYQJTFpHFM0DXmaUDYNTjBdW5VOmjCta3YnE7Z6XdayDNVA7iJUHGVpAwTER4sq19dW8bl2xJhE5lAPrd4ukVonIUaaGuaVrnVrzl0iTalE8XzR0t63uKdICOYlb7XoKDFnS9qXtjnoiiae9KyyTrt2gpg/RjULpF0xz3VrI2wqJe3O0wrbKTzBGpaIWp28sUXSJrSt7xpsoo+kFqL0yCnSvrC3t7dIv0vTdOGMWF1dve7x7r3n4jMVozesutfKocFOUlo7tGk95x1hfFFxmeXFNDM7jghCCJCvq/mrS8utaWYw2zHSlghGrxshZ6vC0csmDQ3OwfA124/uFkwv29XG4BH4jh9r+Npr1S1/tt4p3mlSX13XpGn694BvB/4u8EeAQkT+MfBFVf1nd2dPbx0PLWHPvdU3G911M9R1zRe/+EWSJOF7v/d7b6uKufzFwAs/FUgHJgloFTHdU1afEMojpTxoF9o2bLEuWxOKIyFfNfmhmRmxrzxiGuvxxYbh6Ij1s12Gl+1/vVOWG+IL8EWbG3JkRJOuGkH5NmtjnnMdGqV/2jE7DjR1hKQzFBtWkA4cQcNCHmkKtYkyHSOko9E+SZKyuXLabG6xJ+2ASkAcZGlKpilRZuPB6lAwm86YVQVRHEMcWzORQicxkhYgaTsdnThmdcOwrNiIEyLvGJUl3ahDJ0qJnJHG3AXi3BWCjBKbJTm/2mgKq8RdbPKGr0EaIxNft6PKsEt4ceYfb9qKPQRwPgWBtGcVcDVfdJxZo4tzNlBYsEo5X7GTHKXZLetZIF91i0S/0Jhskq+aZQ9MY08yI0CJbPE2aNPeHrK+Q531+NuQBzuBFJOGfCV7W/rdbDZjd3eXl156ieFwSL/fZ3t7+23ZG4dv1lx8usbPhDQzv7kGcxDFqT2mNvacymNzCvmZMLmspIPWX60mfdiJ0dZMfGG2x3xNKccmcfTO2PP0BWQbdkwXR/PeApgdmI595rvg238YavVEbz48Wditl/s3XPXnX3lne/XO8NAR9t2UQJqm4Rd/8Rdvac7iSdQT5YWfCozeNM3YVtkFtzajuw2Tt6xBpXtKmOwEfCVtmps1oMwOrHLsbCvOCdMdpQwTJtWY1ZUtXBmRbAxxcZ/isM0NWbMPQzUJ9E5bg0eYtSO++tY0Uk+sHb04DnZiGAjjIwiziGRdUG/kFTkjGymtxboplGJcczjcZ9BbZfVUTpSY1IFTvLdKMXg1BwgBguCco5v26XQDaw6KumI8nTIcDhnPCuI4JsQxWZoS2tI2T2K8KkVVo0SoRqgKvTzFeWcnlok140gkaFulipjuHiUW19qUWARqrbjILcjWtdY+u73ZFQmWQwJtY01H8JP58WTvVVNYZT2XWoK3x0syAVr/dtnq7z2TZJKOw1fWWTn3tccdC3eKO+10G7X42WwgNK03XCLAWcSqy7xJPJW21kR7nZO+UpQlvRMZGp1O520T0Mfj8ZUBukclndETRNMV6+SViGoq4Oct8lBPzKOuwRwfSc+yaJoZ5Ot21YdAd9vI1kXt4uKkbYBKdNF9KiltbooQ9e0YbUoj6yiBprBjfutb4Vt+t92uGL078xyvh/daUh88ZIR9M2/1rWI+Z7EsSz796U+zurp6y/fd+1rghb8TKI/s986WEXi2LrBv5t+4D83YrH0rj9gHuTgCUPJVoWmsy6uZCNJTjseHECLWV07ROyOEWhlf6KCJkg+gHMHsQBmctyaQ8vCKS6GeKMUw0DtjGnZxaBNp6plp1HEOWnqbIi6YXS+IXc5PLZWvYsLkqODUI5vk/djGgCUmPcwT+OZBR02lbfa1+aF9Y+PLCBBHEeurq+R1zeoajGdTDsYTjofHFArqIgbdDrUIK3FEJI6qDDhVBkmOOLuKcLEQxVcq3iiVheXOxWKBULE5D1zkFgl6QRWJ3Qnd2PRXnNnQfKO2WDhtNerEW5Z1m3E9934nHaFqq3pfK3EHJAEajIBLq9bFBWiTCX1tC2+qutDb56Sdrcyvasz54lxkenY2Px5t7cLiAkwP1yjQeBb21KshIgwGA3q9Hp3xGXZfV8pxoKlqDi8fELQhSzrknQ5xyJkdmPymlQ2kkAh8IcS5HSPlsTVBJTmM3jTPer4hDF8362f/tHD8qskevdPWxage81TX5hhRB501YfiaFSff/Lvgwz9w5TP6bg3gvR6WhH2fMM8BmQ/wvJ3RXVdjPmex2+2ytrZ2yyEyTRF48aeUS09ZwFF323TpetIu/ImAOopD01/7Z+1DPLncVi1bRrpNaR8ciSzveXhpSH+1w8p2Tpw5yiNzkkhmeR3l0MKf5tp13LERXH5ml+DzobnlcbCxXhOTVyzwyNrg4663xSYvJB1HUweqiTXYHOwcQ1Zy7iMbqJ9frczzrVtHSDciNGFhhZxnYKsHHKAOlbCw7zlMd06SlK31hGZ1hcPpjLIsGY/GFL5hc2OLurTwprwKBB+gMf2ayFw1USKIGLnagqDp1GHu7U5koWEDpB3bfydQlYE4bRP8bG/JVswrvbBzOksajDumXfvSmj3qwsjVZUawqnP/ufmH54sJ3nOlY1Ssq09c+zv2+mcrV+ZzimsXUGHhyplfKbi4neQeQRU3pCrWhVlV183COHqr4uJTDbNDG+ogGtPJU/qDPvVMqeqC8VHJdDjCRQG3N6Dxrc4+tRREaTtGsxXbF2slNxIuh9bpmWRG0Aj0T8PoDXv8zilzK/naqnXnhMlb5mv/rn8Pzl7lBLkf8xzfS0l98BAQ9nxh8aWXXmIwGNyWdHE19vf3efbZZ/mmb/omzp49y6/8yq/cUrfj4UvKc3/TGkZcZnalaQGDR00OKQ4BlCj3pCu24DSfs5iumB1qugv9c0YAs0OhKGbMwhEbp7bIewmLQ1us204bR7we6A7EZi/22jbfmUkF6aoRSDU27RqxsV5JVxa+6KTnKGYCdWLuE8wSmHSFctZwsLfLYCOn39lCa2k73cwlMteAVRRfXolPjbNWRsjAEwjOU4dAQHEBIseiqxGs9byszW65OhgQZxlbChocsRPCxGZo7h/t08165GlG8BEatB1ddoKg0zbjum0ht4W7KzbDudYN1k4/f0F9qYvIWDCS1RBwh3abpjJbX7bqrhC8VwJK3BN8YVXmPPEv6bXBRbRj0zpiTUc+QNDFYmUUC0SBqNXXNbSSBzUuNWeNOGsuUm+hWT5pCBpQSRCU5kQOzhzVpOaNp2xRsZnZgAfnIE5tFmgztViCNHTZ2Oyy1hNmw0CVFIwvwqtf3icZeLI4J087ZL0E8a3Do71aIraBFuotS6TXWph9Db1zdrt6ajkhLrUxaqrWcv6t/yasfeAbC6p7XWFPp9Nb6qF4mPBAE/ZJb3WaprccX3o1VJUXXniBnZ2dt81ZvFm3o6+Vl/9uYO9r1vwC9oHrnbYP2+SSIE7onbKORYJY6FF7TM4v47unreKa7tnfp+xRVRHrq6fpbdkHpThkMScRB1FjkaUipr2WI7OFpX27PA+NEIs9N1s8cvgioKH1cjfWfi4CGnma0k4gAaUOJcfVHuurlnIWpxb2JGqdhGbtaz3JCYtgH5eCJEpZW1TftG6IxDGuSgRhXFUU3tNJEtayjDiKbFoMpl2rQvCeTpoS+wSCcP70JhcuvMUgX6UoS44P9hCNSZOENM+IxRqg4sScGGAkN9eHwTo8dV7tFjbBvSquNMqkK21lfALibRr6vEJO+5Hp+s7IrpoGsty0ahdbBdyUdkIM9ZX7Qfu6REB7KPkmtDZBq7wRFpNxNCiu20Db8R6lV6x/GgcaGmu68Z7UWZVd1TWd9li9/OWS3ecscAmFKDPXkThrdmlaH3gzbmUcb/kt3XWH7HdxOuWRj25x/EZDNakYdo6ZvCgkaUz/VERUdYnjiKjTeqdrOyaS3KQ5MLdRsW8EHXeh2LeKvHsKPvUfmEX1mp+n+yCJnD9//p493r3AA0nYV+dWz7/uJA61KAqefvppVldXv2HO4o0iUYdvBL72NwOTy/aJ627RfrCFcmTuhHTVMhQmO5aJMCJQHilN1DairIhZ5samsUa9ht3Xh3Q6Xc5/qIs4kznAFhXrqS2mdTYhrk2WmGkbcGScj0tNcmmmSlFBOrAPaTMLdDbMSlZNzKoXpeYAwM+zqoXZdEg5qthaO0Pej6nLYPGii25G24ZzQrJuVWaSzxfszCqXxjGV96SRo/YBh524FOimCeOyYmc8YbWTs93p2H2iiElV041jtBG8BLaSARoC6pXOICPvpfT7A3zjqeqKclwyqkeoeDppj7yTk2aJtUkHczvEuSyCn6Bt4FA7OTWFWfOawrzYUWKVMMECrkQjfKVvc3VosBNWZ9WdcHoAKPkai7/5lrRRtUaehoWFD1isGYRW447Slkx7kHm7olK4om0HZRwKsiS+sk2xj2dd18x2hZ0veor5GLN2kY/KEVwgblLr/hwJk7HFG/ipnSgkgqYUkp4i3ZJmCt21mEEWM93pkm9Cul5x8EpDVQ2RwQze6JMmGYNTKWHmmF6yfJa0Lxy1czoG52D4uj2RU5+ET/+YSXbXw9zVda/wXhvACw8gYV/PWx3HMUVR3Na2dnZ2+MpXvsInPvEJTp069Q3/v9ZJIDTKq/9Aee3nzZ3Q3YLpXjvAdrNtz22Ucmj71T+naDDSDqOM7DFFS9uWi1loluOjkvFkzKknBuS9lOLQLGCdTWG232Zib1kudXEIfpySbwRcK2P0TlmlUw0BWgvfxKbCdLYBNXnExRbO07SXqC4Dqoa4oxzs7xNJwtapLTRYwt0i37r1PEcZpKtqI7yckZ96oH3eEiuu7fmOnaPygTxJmNYVSbu2EFTptV7srx8ccL7fx2GVYj+KSWJHqimdbkxTCEmctTqxvVZpGiM+IlvpEKVCMW6oyorh0YiqrsjinDTL6AxSxMUWLFReyeyeS9Rp35lbpG7f26BEIqjY7EraAQNN69BIu1eyq31lV0pxbMSYdFv3R35lUTMEa92u2/22REGHiFozk5h/HMVmWnYdpDbUQRJB6ysLqaVvCF5pfCCeR8uq0kyES894qoOaWFqt2kE9NinFpUAT4VKhGtt+ZAPH+EI7TKJnTSwS2/MRdZarEkAbobPdnoCKjK0PpMRpl3K4QdktqGXKzsVjQOkMUtK4Q2g6dLctbKyZ2gLkuc9YjrW4G68r3Y8Ke0nY7yJOSiBXe6ujKLrhwNart/P8889zeHjIk08+ed2FxasJe3JZ+erfCOaOiCwlr57ByqOKr5TZvu1P2hMLz49tdR2xYQPHx5apsPaoNSzM9ow8JsWQEBeceWILJ87yqleEami3z9chznVRbXfWhdEEqpHQPStEGZTtdO981bKby2E7NDdpYz8TuRJ4FK7kOJAoTAO7Fw/oDTr0Bt024MlkkHpmHYDibBuqfrHgZ5PO2/jUKDLdlwiw9L3aewt1at+mTpIwKkvzXUcR06KwDIuyYjWNScSRuoiyVE4PrINSBGpvZ5coY9FtOf9ZvdLpx+S9GJWOtd6XNVVdsL8zARXSNCXrJMQuayWGNqM7WK6JS8x1ooQrVXMjaOqt4ajt5ZhPQFc/17FBxYg8MG/QsUXKughkfauMk9wt5BolEHXsBKBqAVl1YTklPqnbmZltZd02FXkC02Bnldp7YpfQNMrR1wKzC47JVE0ikxgnlh8jiZG594G0ExNKO1HX3qIH0oFQT8zWma4qvh18EWqz8YXGFkHNgjf/eR4EBf1ejp/lrDxuHvzZpGAynnFYHJg0F3rknQ4f+5czPvIDNybqOe41Yb/Xps3AA0LYt+KtvlVJ5OScxSeffPKGbpJ5xOo8sOmNXwwULSlHmdA7ZRM2xnOtetsmj/vGBtSKSYw0lTlAkrWazoZndmDzj+KBZ++tA9Kkw5nzpxAxcg21QNsEoxIIpVCWLRkfK7NDJep40oHJME3BYkZjcWykEmW2rcifIGo/rwSN8FwEw4OKplBOnVvBhcRiRiMjp3xtLqxq22wCSR7hvQU8KebQiFKrHl3c5mEkStQOJUjjiFndkLiIuTDRTRIaDQS1dD5QdsYTtvMOPsCp3oA4cYukOke8kGGaWVshi13tzJtfTOIwYumsJPRcap2dhaduaoqiYDQc48SR5gndQYb4GNrEQIlY6NxV23wTtVLE3MYXxSZUSARRZFbAeeNN5K5o6L4O5IMr7o8QQrsO0E5Pj0/4uUOwk19kxBwRX1lgjoEaDkpbbJgf9ZOLMHreUY+tQvdTR6PBonC9FQPlgSkmEglaCLSDBsRZV+I8/Kt3Whm/BRqEdN3jL2RMaJP29turhzbwqjxuQ5o24LjtVuydFcYXItT3GAx69GOYHXgaKen/pld5s3eRw3/WW0w/7/f7N+zAnE9ouhd4p40zDyLuO2FfPbrrem92HMc3rbAvXbrE1772tcWcxZvBOcfsIPD0T3iOXrZKsXua1lrXtuqm1gxQjWC8A4Oztig43bFtRInQ3QQEphcTykNh5bRw9FbJ/t6IrcdX6K9liwXHKDVbX1MAzi5LXWIr7sWRkq8DEqgOHX7q6G2Z1FEcmdZK5C1lrWozoktdxI2aLQ3iCKqpMp6MqJmRxCkuJIsJ3+nAKkFfy2I2o3pb4KznC46NEmd2VTMfSiDtd4eziti5RUNM2kalOuxEOC3rRaU9q2u6cUQWRyQk9DoxzdReMxtq0Fg3oLf2fVWb46hKOzxXqaZzWyE4OaEVZ0K+khPq3LrufKBmyujY4mijxJFnHVRyIhfhvbk4Qhxwk8h81e3w4Pl0FbB9i1v3R/AKoc0gmdv7Im82xqpdzRS1jOvoSqTrvJonggp7PaqmuUL0GqhRpk1jMytnCZPnYppDh1a2blGUYXHirL2aF7yxZMfJDqAR+el2CENsIVl+JsSpQmaLk/mGnXSnh0LUCQzOm2tJnKXrlcf2vLOB6d3TPZPSutswvWwVd77WDhwA1j8U8Z0/0mX1iW9C9WNMJhN2d3f52te+xmQyYXV1ddFCf/Lq9n5IIktb312E937RGHAzb/WNKuwQAl/96lcZj8dvm7N4M4ye6/HKP+oz2BCyNaU8Ema7Su+cXeVPdhSmRuS9M4pzMN21fcw3rdFC24pHvVo7bwM7r4zw6YTz37xNfSRMd9oshtI+QNmKth8A80aLM1mknioSOVv1j2zlvTq2phxfBfM8Bwdtp5+LxJpnproI5qknSgieo+keWdRjrbfN4XinDSNScG3zS26Z1fWsXWisLTPEN9pKQuCrYC33TtFIqaXBq1qV7Vho2UlkxK1YZkjlA6rQSS1tTr0nT3IijVjv5hAcUWrSTVMoSZJZDkhlNXqUtLoyLDJF4lRwsV3pzMkaIEmvaMpxLqRxTDzr0U3bhp/QUPkZh6M9xMckSUp3pbVRJg2KtWnHuRDUuvR8jTkw0rm1sD1e27hXlyqIvK0ZRhxIJ7QnPpNfLCNdmVIRuda7rrpIKQQ4KGfWePJSwmwnXbSGR5HJGzUN1dBB7plNGiIiZk1Dd5wTJxGdzDF8E1xmTS/lsenVSQf81K4Co9aK6pKA6zT4yvKrXWTdoPma2SS1sefaOw04y8rubLaupHakV74B3/kjV5wgIkK/36ff7/OBD3wAVV3MXnzqqaeoqmoRYFVV1T2XRJaEfRdwMrc6SZJbWjm+HmHP5yyePXuWT3ziE7fUUFONlef/VuDwpR512erQ2KAAXyvTy61WvWJk6uK2PVets3G2Z5eT3VOCy5Tprs0D9KWjTHfprPTY7J+iPjJte3ZgJ4OkD51zNnEajGDijmV8iFiWdT0LaCNoHRH1zMvsIsF1TQtVb4tUSaedEyi2MFpNFArQpORwf0R/sMZgMzfSTUxeacp2W20rdtKJ8HWwgbaxNQdZep8tKkniUYUq1Eg74dyJMKkqgkLZNByVJRvdLqtp0rpF3CI/JI0iRkVJHjkERyeLSaIEX9rElqYMuNhR1gXa5oOguiBAaKtbM3kAXCHnbD5w4Qp52+SbtpXdWQNM3ktJm4R+dwWJlNmsoCoryrK0ZqRoRNbJ8b6NMpX51B3TbjVAnF3xZ8cdq0znCX7z/ZTcBmeY1m8bcondrmoaInUk80U5Z1co49pz8LJHXu9Cac1XUWonao8ynXqaqUMzTzMWC47KLJN67EqqsfIIa+SbUB3Z8Nt0xbTtYtf84hqgessqZkmVej9jGiyLfXzRJKxsBcpdFqFaLrGpRwCD83D8ov18+jtaJ0h+/c+YiLC+vs76+jof/ehHF7MXd3d3uXDhAru7uxwcHLC9vc36+vq7SuDT6XRh4X2v4J4T9p22l19ryvmFCxd44YUX+PZv/3bW19dvaTu7Xw288JOWtVHNEuJeQ75m7cfTHVpftTDZs9FJnS0j7aawD8qsNNKOsrCotrMVR1EWFOWM1Y0uK/01BJM9pntWmSQDm7M43W0lkamNn0oipXfOolbB5g7GfZjNBFWhe8pRjYKNGMucLTYViibQO2XuFV9DugLHu1Nmsxnbj6zjQkSUmFMgHmc2oCC3KjZyrdNgFki61k0Yd644CaLEKkGHQ/ELrT9y89ZrI+NpXdNLU0ZlyRvHQ84P+vSTxKrrJCYoNL6hn2Ws5CkreY7W2g7+veKiSeOMEOwKJtTWpKKixLGzVT8x//J8Cg7Y++RrBTGniw9zvbutiL2SD+yK4QqE/koHkS6TUUFZlERxxKwacjxsSBOLjc3yhLwfn3CX2FQdF7UBTnMpR+11irsQ2mEIXkO7sGhumbEJ4fgQSOOE4D1OHJM94fUvesK4Y0TengRkJoQo4BtrtW8koAcJPqvQsn3OEVQzT5Yl5CtCPRPirk0mLw5YSHuzXaugk75Vz5PL9nqmPWH4qj23/lkYvmk/5+vWAVkc23vT3YTRBZNTPvCb4BO/++ZOkKtxcvZiXdecPXuWpmm4cOECX/7yl8mybPH/lZWVO+5gvhbudWflvcA9I+z56K65jnW7L+RJl8h8zmJVVbc8ZKApAl//Sbj4BdMgu6dg9jqLaeDWhGCdhJPLRqJyUqtOhc5WSxxF6+DYsukwBxcn1MmItfMJOs2ptF0c2zTHQtS2nOdrwvRAKY+NmC2oyaqZtG+BPL7Euu62aqqhozhuNVdvrdMuVbrbJoMUQ4tJLYeB3TeGSF6zsb1O1oktenRmum+DVZO+aMmy9RC7VHFxWGjbLjKNWr0RyLzTMXGOEsjimGlV00kTirrBYRr1uKpII8dRWTKuKjpxRBLFTKqSPIpxCIMsw6mz0VuNI3jLtg7txBPEJJI5vSaLtnIFZ1bCKLbqXzUYEasR43xMmbR5Ijayy6bYAK2rw5OkzqbhoNZGXsHKRg8NXbNMlg3ltOR4us+4dDiJydOOxSFEisvbZh1tO/0KtXUQrcgkgpbAxVmn5jQ01BoWi4k+WCb55Pk+46FHmhgXIKgQCsGXNrszVEIVPKEStAYGDc2OfVRltcEdJ9aMtB7blZ9YF25ow7xwSiiFfMOGJjczUBU6eUN9YDM6++db2auE/rlWNplZhZ9vYfMmGyP0D/xG+OC/+M6J1HtPnuesrKxw7tw54EoC4Ysvvvi2BMLt7e135PB4L05Mh3tE2FdLIHdyFp1LIqPRiKeeeorHHnvspnMW5zh8Ufna/9x27bULMeNLSu+sp5z6hSTiYmvBlUiZ7tl2u1uWzTFPJgvY4pQvYXxZGVV7dDaFQTjNdDQl7oRFxrU4m5WnAdSLWfhWBGJbwJsdKp11i76sxtYZ1zstNhFmlOCyChe14fKxVfa+FJusvmITZaaHDUfTHbrpKhtnV3GxUhceQRaTz6XJWv9waBPYZDHM1vtWgojmwU5GplFk34na6efO4YORj2m0SjdNmVkEHd0kYdY0HFc1W70uKzHMyopMHJ00Jo/TlqRtXuN8pJeqWdSi2Gx4catTq28XVWs7wc9zrn0dcKnlYLisXWhNrzS5QGvD81daG30dyAYRerLaDiB5feWDrZBmCVkvxrkuqkpVV8xGJePxGNKaTujQSTqkibltkjSiwtvxHQKxzP3T9vofzGaAmmumgvFLSnkhxs+cjR6rzU4p3l5PglAOTV6pikBQJURKmEbQaawCv5xC3jbdXE4pOtbMNdvD7KVrlsCHg7SvbWyCyTPNFLSM8GU7UPfQXv90AOO37DWP2hS+8sgyQT71o3D2U3en6r3WouPVCYSj0Yjd3V2+9KUvMZvNWF9fX0TI3onD5G5W7A8C3nXCvluju5xzlGXJP//n/5zv+I7vuKWEvdAoL/+ccvGfB6pje1yXCv1HbGHt+GJCVSlrH7CKNUrb7jHEprocmqTR2RLijjK5ZBUrAm5Qsr+zz/rqOonvmLZdWpWifWVw3ibCBG+SQ75hlXCUO+qJjQ2rZzA7NIeHy6yKLA6tEp9NwU8jpCNkA0tzq0YWYFSNlWqo+Kjg6GjE1vkNOv2Mpgz4Uog7Dl/bPMi0214+tBX/PMZTxCrEJLep5nE814+xBada7ejwoM6aZDyQJTGzuiYWm8nog9KJreIrm4Y4clSNZ9/bDMc4idjs9mzRrU3NUw2IugUJ13VtjyeAE0IVrgwayJ2F/+fWEu7ieVQqbSLeFSuetkMOVEw2iBNboJxPQBe50o0oaSDRxCyLlb1EOCXJWThFsjQj386tpT6uqaqK4WSIeIjiGBdHkNmg56a1pM6r7FFdGwkDs4vC7KWYZiqIF+oCqgryvPWDO0XHzvK8s3YIrghBFBrBJ40tyqoSbVY0kwgtHN2tCKmFyc4VTf/4jdb9sXlF9uieMatgMYvQxHoLJpdb298p4fiV9nbbJqn40hbVP/OHr50Jcqe4WaejiLCyssLKygof+tCHCCEsFjBfeeUVvPdsbW2xtbXF5ubmdUOxgDuOsXjQ8a4TtojcldzqZ599lqZp+L7v+74bvlFzjC4EvvJXlfFlI6feKatk83Ub2RXnNlGjvKRM95V+6wyZ7duleT2xlDyXKtNdC3tPV2yx6HhvQt0MObV1irQbUx63VqhYiNcrgu8w3bWFwSSGcmSWtXzDCDM0djLI1mxqTNJ1Nvy0tYwWR6Y9NzStFxvS1DilGFq28sHlEY1MOfuhTWgsDzrOjNyamS5IW2KBbgFYR54NtbUgJdVAU7IIdrKgfZs+rtgJxIfWzoe5QtrcOdLYiFuwnJBRaVptHscEVUbTGRt5xkanQy9NUALa2MlDrPeGODObnojp6wRAWymjlUDmpA6tnU4gFqEuA3E7rV2VVns3eSAstGxzwCC6yBLRYJnVvmoDmbxttymVrGuVrkul9WBjXYqpx+HI85w8z3HqaKqaS+MjilFBGsckSUKVZQw6HVSEy5MJySSmfjkjDCOonAVpNWbjU++YzKvoWsiSiFA4wp7Dd2v82OERNGtoSjs2vWugjJHYI2tKmjpIhGxVaaY2lKF32nznc9KdZ4Dka+DWayb7kHSwCFdM2+6fb2czlqZ9d7fhkz98/UyQO4X3/pY+u3M459jY2GBjY4OPfexjFhK2v8/u7i7PPfcccRwv/N9ra2tv45iqqu6p5/te4Z4Q9ju5LDk+Pubpp5/mgx/8IMPh8KZvuAZrLX/l55RsjTYtT5gdKt1tbAHLK9VIqOuIdKukvzlYaNVJVxatu00l+NGV6ebFkTIJ+yQryrqcQSuhrOzADk0glFAeJqw8ZtVwPTOZY3DetO5mJm8bRqCqJAMj8+BZNMXUk2Cr9V7bJDdbXMxWhNmx5/LFA/JVx0Z3m1BYE40TXVSTtLp03DV9NyZfBDz5qvXqzlP5Gmutj1IjKpcoKt40VFVmTY1Tx6goSRO7QsqcI5bIbHxJTB1MEkgjy7ieNBUO6Ge20NguE16pfr27UgF5IU0yI87YTjJRKrhIWkax2FHrXLzSdp7kDonaQQa0bfjzpQyZx6LacwLzuwdv1sM0F6oQkHZRUb2l+QX1i8XEuY86RIGqaciTZGFVkQhIItJul7zXI4simrqmLEtm4zGX9mdEOxv4cY56G8+VpLY43EigOna4TPHtesGsqRhPI6oysNpP8fuJZXv3A/XllEYD8VpNOEpwsSN0G9JpRojn6y4sTloilnAoCcStvz+KzSUy2W07vcQkunpiC535hlXm2sCpb7+5E+RO8U6zROI4ftv087Is2dvb4/XXX+eZZ56h2+2ytbVFnudEUXRXuhxF5AlsyszXgEpVf9M73ug7wH1vnLkeVJVXX32V119/ne/6ru9iMBjw8ssv3/A+0z3l+f8lMDswApzt2/G5+rhSjmCy09r1+kKUKakE9i/EzJwljc32rnQVqlraXfAw2VUkbxjOLtGVbfKQE3WsQiyOgw0arW1lHZTiYN4pabLHdLdN4RP7AJUjpXvaxl3VI0BMBimOjFDzdWE6hlDbENlsxZwik6Oa4/oSa6tbZKkF/ovQOi4sh8RsZ+YR1nbKOWqNJ/Po1eBZDABI+qDOqlUbAyY0QcnaCjpPYorG081SZnXDuKpI4oiirtnqdEmiiKKpzPKYJFTeU1U13SgijWPyOKYKgRQjeKcRjbfuUifm7gAWY73m7z1iBIuzPA+baNK2QGL7HzxtWJVlYC9OAmJXGxZd2v4tMjddFNvEHOccRNaTLk7wrjZLXludK4okUAQbv157T9wWHz4Ejis7UwRVpE2TdJIyueDIXgtom7aIeLQRGhXqGsopSKehPk7wEbg8oKWDCLwEjsc1Ia/oagrjCM0bHAG/m6I0kCt6KUfiGLchDF+z16x7SphdsoXCuGcXFeNjoD22j16GonDEK8JsV1pfdkvWbZjTB/8l+LZ/6/adILeKEMJdDX/Ksozz588vEvnmDTx/+k//aX7iJ34C5xx/+S//Zb7/+7+fRx99R4N/P6+qv+eu7PQ7xANJ2PM5i2mafsOcRVW9ZsX+1j8LPP+TAV/Y/zpbVjnFXWH0ljVE9E7DdNcqSucgqIO4Qb25QbpbNr9usjNv2oDetnB8MGZyPGalc5aVMwmzQ6tqxNng3KLtFKvLCNcriTuBtO+YHUJnHbRR6qm0rcJQTUzfdom0XZRKcRzonRKqmZ1cQuOQrLZut+NAyIeMj2dsrpwh68Wg1hIfxdbIkeSu5TKrtKPEfLtNqXhtcypmgXzFKlOJFfAE79rmlNYtEtQ6AoMnjiKqurbmCxehWtNPU6Z1zaiqqHxgq9slqNJNk9abXaIaWMm7bHW7bUxoDO2kdNRmC4Jp2FEG6mpboM3bqSvKYio5tOH+TVt5O9PNQ5v3EbxaDkbbuj7v+xZnw2WbYv6ymP4t6RX7XxTHJg1FFpfbeOte1JbjJ6FuxSB7zDg2spk0DVVoc1SAsm6o3oyZveGYjS3/OkiMqhDGDo0b6sYabyZ1iUxSPBa45acRIW4J3jtKbQgeVGr6g4DOgAZko0QLRasYWanpdCPKoXWtpiu26OgSS3os24nl+br9rR7bcGhHRTUU8jVbXAzBdP+Vx82298Hf/O4v0L2bi4C9Xo9er8ef+TN/ht/7e38v/8l/8p+wt7fHD//wD/M93/M9/PE//sfvdNP/Qjt492+p6p+9i7t823jgCPvw8JBnnnmGj3zkI9+QZTt3ipyURcqh8rW/oex91SSDtNeGNLWXz4Jd7vtKmFyGlUetCpnuQfCOUDi6j9sl92wf9MgO7qawNu29/T1CI5w+e4p6LEz3lChTOmeEYl+Z7ggutUXK6jIstNYWs0O7vMz6Fk86O7BV/XllbBNlTE8tjq5kT9fHQpjGZKeV/f1DOHZsnzqFcMXDHGUmgUiEdT9WuhiLNZ9/aLMDI/I12mzqYFPIG4jTyGI8vc1JDI0Ymavtf4RN7fAhEKUZceQQcXgN9JIUBV48OOTx1RWyKGJS1/gQyCOrrCOxRpqA5XTHxG31bJffYBKWtt+DWoUvQXG5M2uZV3xjDKpqt40SLDO6DAhC0nZtxh0hVFb1urSVnNp2cqQ9OWCZJXOoCza7sj0/zH3UDVA2NXmSIO3/ah9InGNvMkVRBllGuaNUr8fo1BG8TQgnErPklYLLAqGI0MiyVZzGaK+hOXRQOqpOiRylNF7RlZpmGNmxGNfETQQqqAv4pnWgJB4XCXEUWUZ5u5A7n2IUKkjaXhHnxNYoMot7HV0QNDhrhppaNLA4+K7/4O45QR4UzGYzHn30Uf7gH/yD/ME/+AffyaYuAh8FSuB/FZG/r6rP3pWdvAM8MIStqrz00ktcvHiRT3/609fUn+bNM3PC3nlWeeHvhIUUUQ6til19wjzK9Vgsizq1oQOIML5k2+puQjUWpLRL1mo2n/RsX1GvYcglsmqDXt6lGtnKe2jMsTG9bKl61cw8r9VI6Z5WhrumDdYTW+AsJ76dnyhkGSCthS+CbM2qvvLYgvHtxMJiZFMhJTtvNnRXUjqDHr6yfO2oTQp0ybxD0bYbd6yDcT6sIIrB5UIyNSeEadbmYZ4HQM3nMsZZm8YXRa0XO3B4fESaJIhzDKczJpMxjYvwzpHlOdO2kWl3NmOr06H2Ht80pEnCmZUBpff0IvMnR61PWaI2sjUGtLUMSrzwX89pQ7214kepIGLBStb+b7GmSrugqFekEA1tamHkFul6QdurD0vksMeI7DdFKbQBr6Ttpbqqos4xLGykTFE3dNrjLahyVFe2pSFMLsVU+w5tHC5AOYMGxbWj1oKHei/C9Rt8KZRty35dCJJDyBpklBAipcpK3E4KKGGlRvZypk7obAWandSm7gxqmMaEOkI3IupcKA7sXNM/w8IV0tmC8tiuOFxsYU5HL0I5jci3lWnb1Zivw5N/5O46QR4U3K2kPlUtMbJGRH4K+Bbg/U3YZVny9NNP0+/3+dznPnddnWvePBNpyks/q1z4JcviANOdk67NTBy9CTild9pS9PI1oRoJUcIJu57SOwuuqqjGinobrZRvKqWfsPfmjNXBGbobCS6xlDyJQCsjTLDq2SU2v7EcCcVhRPBiTpRDkyPyVYfSjgY7sKxlMB+sL03SENdOh4lNU45SYXhYURQFW6tnSOIE58Q6EZ05PSS2MVW+sm3WM6WZWtddlFilbFkR1rCEmD4/J20iAcJiuK6vbJiuD4FZMeHw+IjNrQ3iJKH0nu08Z3V1laPplOlsxvHhIQdliThH1OnwVtOw1e0hCBsteXdik26sPzGgTsxp4rXtFgSCyRlxJub1bpQQrJkmisy1ojqfdu6o52FL0IYsKagtVkK7IBh7Ip03yRg5N9IQqywWFcUF6tDgWv0jiODaRcVJXS1OIKDWbNn+YWd/SnwhJ+wnNLUg2l4dFUoV1TQ1lM6jMwe1o9NTit0YyQIhVkLh8FovrnK000AtSOWQdY+vG9IiocormtgTH3SRKBBWS8J+ShbH1IMKpz3KIUQd66Sd7nBl1uieXYVkayaNhNrcH1qXUKf0Ttvvn/xhkwHfixiNRnclqU9EBqraztrhc8Cfe8cbfQe474S9t7fHl770JT7+8Y9z5syZG942iiIOXw68+hOB4tAaQPKBhTQlPdPx8g1Ieko9sa7C3ikAy7P2pcAYemdt+Oh0F5qjnOhRc4Y0pXJ4cEQ9Uc5/eJPyyDI60r5lKhRHJq3UMyN+9eYPnu5Zu3nsQScWwdo/a1W2DTpQOmttA47aicXFVomXQ3Nr4EyWEacMR0dMxzWZ65N30zb0Se1S11kjDSVkffOPV2NLF4zawCOX0CbwmVzjSGyB74SNT9UysV1iLelVaAjOsXt0xHQyobu2xhgYDUdkSUw3SVAN9LKMPE2RLCWuKpq6YTiZMDo+Zpd9zq6tstKGPgWFomnopsmVBpU5iaqRstISt5gcQUQrvchCCoG2aUk9cSqozt0krZIh1gRUl34+pAWV0E5UV7wEmhBwccx8NSSgzELN3PhVNU0bBQsHsxmdOGGunJRNQ9xE7L8U0Es9go8sICs2MlSxCed1hblKpgFxnlmpHA0j8tyRFgn1LCF0K/zMIRLRRBXBu9ZyaJ2m1EJnJaIsanIyJPUoFpErg5IoqyhrZzp0gslGtQ2xiGKoZ0K20macBGhq06qzPsxeiImI6H4QvvsPvDtOkOvhXnce3sVo1V8jIn8Cq7L/sar+yt3Y6J3ivhG2qvL1r3+dvb09PvOZz9DpdG56nziOSTcatr9FePOXzHsqKCuPWNu1BiNtERg8GqjGXAlyGtiQWZfZ5aIG6G3D/mGwtLtOw/5wl048oLvSp9i31fZ8zbrFJjvWttvZMMnERfNsYqvgymOx3Ld+Qba6wmxPcInYZJihVejdbasubTKITVtvCus0TFdB8ey9dUwURWyeXuPw4JimUjrr8+wPex3SvqOaBsqxtsN5W1uX09bnbXMNmzoQaiFoQ10FsoHDRQqRBdcraj5tIHKOg8NDCIHt7W0arNq2xDvljeMhkXNsdbt4b919vSyjiRMqEdJuD20asjhheHzM0Hv6vR6dLCONnA2lEIF2GLBTu4pyvs2VvmIAMY1bWmsj0mrzdrKzz72NNNMQWra3qNIotw7OxcKWmHxVt0lNc1IWYFzVNFdxiFdlWtcWCtXmpISgFG8IzVsRRdlaEhvrtKxbf3R5JFTeExyEyuHxuKbV7rs1xyMH3rO2GuF3YvABNjxhJzXbYr+E45RIEoinuDrDHSuuG6OJJxwnEBRZr4gPVvDTggujyzBLiemQd1P62zHDV+1599pGGd86l/I1OHwBmpHj/G+uefIPdN41J8j1cD8mpt8NwlbVnwF+5p3v0d3BfSHsoih46qmnWF9f58knn7xlq08URUja8LF/xfHor1Fe/XuBnS8Lk7kuvW0RpdkKTC6aP7l7yipwX12x1rlIadoOMddpiDamXHp5xsrKFlmSWHTq1DTh6a7Q3RTKUcBX1g4+OGeadVOaTpitWCedy2H8ZoqLLO3OF5at3d22Kqc4trFgNowgmL1vy+JSp4c1x8fHrG53SF3XEu3ims6GPSa0FfVsXlGbx1sRnIO6ClQzjKhLTzULJF1rpNG4IVsFgjXSmLdbUGfVWZCGw6NDsm6H9U6HWdOQOcfYe3ppStF469oT4dXjY870enSSFK+Bom4IwfzYlQaeOLUFQekmMUVVUVc1l/f2CU3DyqBPlqakabpwWBBBo7UtdorDN23YVEumASXqtLp3y7AiGClGcmW4rtgiYRxFRCecRJOmImrT8QAqb+6X0s/Dmlp1qP3fsDSrnypMLzqaC0KYWo62NuYZ0VoX3vlQA7lS7gA+xvdKdBrRRGo6ehmhiaek4nCc0B3Y+DU/TQiJR7sevxdbI9F2gAsJUScmZIGEiPowopEGGTTIbpfOSgc2PPnxOj73+KRgWh5x+HUlzWNWziQU4w7ZamyDdBsgwMpjsP5N+3zTb+/ec7KGez/PcTqdsrm5ec8e717hnhP2zeYs3ggnI1a728I3/1DE2ZeVF/92YPSWLaJFGSBWAfnStL2VR6wFeLZv25FI6G6bY2L/JeHgtZKzH1xHa5uCIiLtduz2031zfvTPWiU93bXK2vRwq/ziVFBnmms5ZLGoGCW2qOhSGy1WTWwYQWfD7HXVKFAUBePmgLXVU8QhJukLWSqM30yoRlaJVxNdVNQSWyaHiOVw17Mr2nQ1s7mDUayWfxxDNsnQZh72YwuN87yQsig4PN5n+/QWaZZSeWsSKeqaJIpovDXQDNKMMnjKpuG14yFn+z3m/uQkclTttPS68fTTlFnj6WY5ZJnldESOqjJd/ujggDzPybKs/UpRrHXcOjCtYUjErixCqzO7xKFNQNsmGt96uUWhaR0ktfdIK300qtRtFb6QQlQ5ns8GFau6u4kteh6XpeV97wtczAjjCG3zPmZFS6AjIcptjBxAE4INOk68uTn2EjT1VGWAOsVLQ+MUqSNm4hFREo3RzEPUQOPQlRovgUj6hLUZq6d77Fz05H1BVmqqWY16ITvjWdmEMEzonwWJI9T3CXXfZmJSMNotmE33UPGkrkPsu2RZxqf+A2GvMyKK7k8+9O12Ob5TjMdjHn/88Xv2ePcK9+wVDCHw3HPPcXR0dMM5izfCtTKx1z8ofOpHHZefUl7+vIUulcemz3XXrallcvmKN7saW6XVNJ7dSweQ1az3T1MdW37w4BGY7Fi+NVgXoy8DcVcW0aghUppCKI6U/hlLAqwmbZt1VtvE9FwsXzhqJ5lUUFVKvmHEWY+tKWRcHdIUwsbqWfIVh0st1rWplBBZc0Y1Miugikdaa10I2g7OtcXIprD27jiz+E+XmAYcGqEJtc1lbC12TWULk8PDEbNqwvb2aZIkRkMgarVuAbIoomoa0sgWTsdVReyE2EXsTmds5hmCBT/tjkZ8+PRpXOSofLOocAUxLVkgTo2gV1dXaZqGsig4OjqiaRrG4zFJmpKkKUBbjbWhJ/Nqu9W45wQ+/5uK0JzIjrB9thRBMBKP2pyPAByVJdEJTbVRa7w5uFyRXO7gRin41hpa249VqdZxmSrF0Ca6aKTmMIprQmk+a12p8YcRWsbooKE5ihGfEDolMk2oaocbgOAI04TgApqCm6TQseYiPUiJRglJN6YcKnmZMK4qoo2Io5dBNSOcESYXrIKO2qlIs52cmJyz52D0llKOKqp8Sv+3PMMrTaA5bFhdXWVjY+OehyItJ6bfHdwTwp5Opzz11FOcOnXqpnMWb4TrjQkTEc58l7D9Lcprv6C88U+UuGeShdM2oH1ogwd6p4WimXHxlTEr/VVGoxH5o+Y2KIdm+0u6jijRxUgkIllU24tZeaeUulSm+5Yul68Ls0PQ2pH0THappibFuERIV9pL+QYkKMSB/Z1jsjRj+3y3HU8FvrCp1vVU0Col6bXBRXH7IfeKVqZbR+3iYpxCskqbfmdE1xS6qLojTS10v1aizIEEdg92iTPHqfPbNBqY+YrS+7aTzxM5h1PFYfkg48pKyn6WUfnAtCwIGhikads4k+IiRyImP6RRRNkm+WVx3CoSCu0wX0To9PvkvR47ly6RJAmz2YzReGzDHNKUOM1Ik6Sd1oJ1bAbL30ijGNQaXqa1DVaITxxXk6uOk8p7Muc4KApQ+z2J2kaYo4bi1YjocIC2sngT7MpFYs901tCEQFMGdOKo05p8lJJEDsmswlYXaGKP1BGh46nzhniWQBzwvQbdj5EghH5NPc6IQgxZDY2D/RiJIN0W3Ms5YUOgHwiHCVpApwclNdFhDqldMU4utUS9aX0FoTFJMOlZKt/gnHD6VMYnf19Gd+szzGYzvvCFL3DhwgVefPHFt43xupX1o3eK+zEe7L02gBfuEWHv7u7y8Y9//B1rSjcbxBtlwgd/k3D2U4FXPg+7X1FoF+r6ZxR1sPvqhKqqOHVunbQTMS4DElkudWddmB5Yyl1ILRSnOAqLSdPZShso1LNskqQ3n/xi9sHeacdh2SyIfp5xrWr+YHHgp0oxrhmOjtk42yeNM8vJaGQhWYQG0oGDQ3M9RJGFIM2nzUSZ3c61uSS+Mi/4lWG55jZp2jS++YJk3BW81uwf7rK+uUqn16UJlrBX1E3b/FJZNV2UHJclvSSll1oYfz9NqUJgXFX4EGiCowyBoiw51+9TNraNyntcWxh3ksQ8xEAaxwvPtIgs3ksXx7g0ZZDlbRaGpyhLjkfDhaa90u2S5Nmilb3y9lh1S+BB1bbT6tfDWUEWx8RuviipFCFQN/aYZQhEpWP6RkRz5GgaxXmzHVYzs28GF9BCqNqxad4HyAP1XkTjGqqkZnXYw0eKpoJOI4gULx4tLTSqiRsSH1P3K0KkpD6m7BZ0ujE6Mz966NRkLiaRCLfRMHhEcBcca+ciXFOSkrDOKolzZucsbHSXzQRtrXsB8j6M3xSaGk59K3z6P7SgJ7AY0263y0c/+lH6/T7Hx8fs7u7y9NNPU9f1YozXzVLw7hT3Y2L6ssK+Q9wtLSmOY6r2MvdG6Gw4vvl3wuHLyks/GwiNUgyVg/0j8nVY8+uEEtxASAYN9cwW4aZ7StJzpP1gXY2XrTrurJvnGjHClLYFvB4LOOhsmsVhdgCUCfkjQjEMlG1wVBTbgqMCRT1hWo5Z39ii27eJ4k2lpjE7I/l6ah5qyRt8ZQFBJ4naZkAKTRWoCwtCasqAlibF+No840nHtHPJa4iUYlYwmh6zsb5J3kmt+9BFNMGmn1eNLciFYN7t1SxjWFZcGA05NxjYQN2mofGBLI7biek1jfe4yJEnCZOqZjXPCN5bl6PqXF2iUV0YQSKxuNA5TNKwRdk0iuh0u8St/l03NZNZweTwgCROSLOULM0gS6lOnMDnLpDZnJSbhiRLF5ayS+MJaeQIU4gv9PDDFA0xdQ1NFYgioWpjXqtZRAiCz2qkEhr15vn3jjCo0GlEOE7YSces1l04TtCsRouIMI6RtCbEgkxToiQiRDXJOCcWGMcF6VGGBvCtjh2RohrhnI3uSkYZugF6Oaf2Ds1Bc1kMc+6fhfGb1loeZTZx6PAle3U/8JvNYy1XKRDzLA8RYW1tjbW1NT7ykY/QNA0HBwfs7OwsUvC2t7c5deoUq6urd0U+uR+E/V6b5wgPgA/7dnBy6sytYP2Dwnf9iOOFnx/ytZ+Z0MtWyZqMfF1J+srkMuikQ5MqvS1HOQokXZjtWwWjI8sIMa1aqKaWuFdPIemZ0TXpCPXEqtu0B+xb9d3ZtEad4sjIIukreztHuBBx9tFTxKlQjq3ajGIb8VRPjdSzNRsh5poMUZvc7isA655sCqWe2mispgzUs0DaznwMjZINWvNE2yQSS8poNCL4wObGFlHiFvP7LFfFtb5oyFzExFf0soyyafAh0EtTDoqC9U5OEwJZZElK3gemdcPpri1ANo0nkivVbMcZoYPJIkX7cyRC3c4/dKrUdW2LjM7WASrvr7g3BJv7GSdEnQ5Z5KjLillR8NbOZTSKWOv3ydqEttJ7JidO6kXTkDqHijAZ1yTDAWE3hsomvWgj1JVHRVvJyTLNcQGtbRiAZN4yslWotCbUgneeeqXEjx1DCvprEX4/wgVH0y1gEiGTiBDXZFFMtNshThzSr4h2uyRZRj2YwV5KTES8EhG7CLeX0wxg7XTM6A2TwbI1kzyqiWnVnQ2T5jpb1oAVGiAIK4/DE98PH/6Ba38erhe+FMcxp06dWpgAiqJgd3eXl156iePjY1ZWVhYEfqfyyb229S017AcAN5NEroaq8vIrL3M5eYtf/x9/Fzu/kjF83YKbJpfMrnc8MZLTYBnB8yzl2b4tzHVPWxfidNcqls46zI7s9klHbGJ6CNQzsUl+ncKG4o6VGrPwTQ4a9l87or+ZsXGmT3FkmdVJxxpHfGXNLp2t1j5Y2ggxdY2F2qt1azZV28jTtQGz9QTirsNXJgtkK84m0Vc2Bdw3AYJSVQXOddnY3mhnMlqUKN6B2IzBKniyJKaoG9IoplFrNukmCY0GjsqSF/YPOT/omdXPN4xqC0fqZSmRs3S8uB3WG0c2oQYgdm6xKKiq0P7ehGASRGsbnNdxdVAabzp6GkWAUjSWvV2FQKeTk3c6hCylaRqquqYcDgkhMG08aZ7Ty3Mbj+YDzscMX/P0dlcpywgXrEknVG0DVRtnm0TBdObY48eOphBCGtCDlKCe0K8JkwhEKaWG0uGjQKkNcZET90GjBp0q9BrqqKSZQtbp0qzP6CU5WgqslqxsZ+ztC2G1oruSkfuIvCt0zwVW1sGXHXi0DSlrbDpP6FmXa3lkEoiLbF2mGtlEmU//KJz7zPU/D7e68JfnOY8++iiPPvroYgrMzs4OX/ziFynLko2NDU6dOsXm5uYtjeebP/ZSEnnneM8S9jzxL8syPvvZzxJFEau/FcYXlZd/zjN6ywYXiED3rKc8sgGl8+kwzTSQrThm++azFrHqerpvzpDQQDUCML92lGDt1tPIcjs6Jm0cXy4YlwecemITaRKKw3nmtVLPbMGwu9W6W47a+Y1tAL02Vl35yk4Aac8Cj6qpySC+sSCnaOBQ72lKacd+BRvUG3uO9o5AIwaDnk1cyc2mRiSU3qahF3UJCEVZ2WSg1p6YJzFOPGXZUDeePI6Z1A2reU5dBXwIDJKUxnt6ScKkLBlk5vLIopjGe7NYuivknbTTWeYQEUIUUcxzR5ArsztDoMQGKSwyRlTbTI72ZBDHEMf0UpM+RvsHHE+mzMYjYhI6o1WKY6hLiw2Q1inivVJX7ZpCWsPU0WhloUohIk4sc0VHjiZuTCfeS9GkQVWIhh185PFxTTTLaBJHtKqE4wgJoJlHRzHiA5mLcHVErCl1U+EiJZ100FmF5B7dT0BSihoG9Dhq404H54Xj18wTHneMrMev2/96Z23hMTSWvvfkH4H1D9/4M6Gqt+3UODkF5sMf/vBiCvrOzg7PP//824bsrq+vX1c+udeEPZvN7sli6r3GQ0XY13OJXI350INrJf71zwrf+m9E7DyrXHo6MHnDMb0s9NbNF10OrZqJ8ysHdjmknVpj1q7ZQWv5W5831NjqvDjAO5qZ6aDTcETlK06fOQ2ls0CnkVIOTVvurNmUj+LYPNpNEahn1p2WrkI08dSTlqiLQDWx/QoaTNNOLeTCl5YEKG4+xMAxKwqGB8dsnlllMhvaCLJsnsvRdn/GscWfRjGV9+agU2V3MiMA/TQhtIt6g9zcIYVvGJUltfd005R+S9CjsmQly1pSEIqmaT3aEXVpi5CRswBva9Y3MqhavTmoMqlqIufaavtK2+OorMjjKx/2oMqkbhYNL2DWvTooWZqRuYzOYRe/FyGVSSyhFpRAXTgbM1a0melSwUFKE9dUKI0qe82YtaSDU0F7AT8FcNSDAh3HSOPwnRIpY+JhRkgacNBcTAmihH6N7OREqjSDAj1MiY4y4s0IXzui/S66HuE6gWSvi0tjutsRoQCdxkgM/VM29TztQbJyZYzZfGBuU9qxnG/Ad/wIbQTDjXE3rHUnCRosB2h3d5dXX32VZ555hl6vx6lTp75hiO69JmxVfeAmpkt7NtN30Kf/UBH2zSpsVeX111/n1VdfXQw9uBZEhNPfLmx+TBj99YLwVpdqrLhIGZw3DVrbuX7piqA+kPRtgnmcC2kfqjGtVm3t3uVx+x44JeoG9i4d0FmPObV2ysL32/jUtO8gbqWAkVXb5cTIOErEiF+EZqoW+ym6IGoN7QDadsp4Uxk5ziehizPP92h6zKyccPr8NpGLmExNq8aLTdSuQBLzHjsLqUaATpwwqkq6SUIdPBdHY1Ln6LXukCZUFE3D0azgzKBv1er/n73/Do5sy+87wc8516aH9yhf9Wy9ev5Vvcd2bLKbThJFrbSMGc1ql5JiGCtRs5JmuaI2JlYMzTCGMQopVrE7XK0Y4kqURhyNKNGIarLZTb62z5X33qEKHkikz+vO2T9OZhaAAqoAFMq87v5FvKhXBeBmIvPm7/7u9/c1SuO7NmELsqhEETnPI2q9T5aULfGNwa7rLXzZs23sVuvuiB4taVLXW+kuApMeHitFpGQrQIAOTl3wvc5CMVKKUiXEnvORJZs4kEhhMHGlNLppEcca7cQEdSN9j5rCUD/TEfGihVQWQSpAVB1qToL0NHZsIs+Ek5A0ADdG+xrVNP4cdb8OocSybcJCHaksiCUqZ/j4QVmQHZFoK8B1UiRaY/sR+UGYmRZkR2Coy0XHAtKmAUvbUPPcnHmfkoZhIiWRwa+Ll80LNnAI3v5vW7uTDdR2BwiACREYGxtjbGzsvhDdZrNJd3c3AwMDhGG4Je3FVupZS0wXQgjdqkc91vdMw27nPgK89957G6Im2b6g560KGZ2iftGjsWCUkdIReN2GqqeVoQt2mCF1w8lO9QBSdxLX/S4DgRBJZuen6RnqwiFtDPRbcvSoYRpu3ASr5VkclE3+IpbCdqVZIGbNslDGHpZjoVXSyiuUBpoJDc3MSZssQt0EJyPA0hQXi0hbMdA3iMTI3i1aZkyJRiJRUiFjAbbxdw7iGL8VrpuyHUKVmLQZ2yaIY4rNJhnHJYhj4iTBd2yCOEFi0tRjlZByXErNACkgaS0CnFazBvMhSpZBIVEcU1OKtOMaGbrWBK27p0YUkXIcJIJGSyoexDG26yKEoNSst74vwbMEKhAks2ZylbFF0lIhqkSY49YEltAkCRDbKJlAaBHokERAXBHEtrmTSBYctK1MlNe8JO1LRCqBkg1aoNIRumwjtMLJCUTZwsLC8gTNuialDQUSV+GHWRqqYfy7Kw5WxoIApLCpToJTSdE97lG+YaiXbgtyDZbMn9lRqNxquxKaBl65JRA27PkyHPw5g2Nvph6nYGatEN02fHLnzh0sy6JerzMwMHBfBuPjej7PQmmttRDiEPACkAKawDzGa3tCa13a6LE+VQ277Ye9uiqVCsePH2fXrl2bphBaloWbj9jxMxZzZzUT301Igpbj3oixMG2bNTkpw9iQlmnOQi4zd1qCUFSJ/Qr9mXGsyEa4AjcDYdWoMZyUSVJRiXHu87KGmmd5JhTWDLtm8nZSEk2MtLXBWqSh9CGMy17UTFrxYYa/HTVjFsqz5DIF0um0yW1MNFKbuDAVgUxJpGOed6QSQBCGkYkFi2Mj4RYmjTzlONiWRZQkzNUbuJaFbVnkWsGm1TCgP502DBNEx9Ij63mG5icEtm2RtPBm17ZWUPCktNCJohaGNMPQBBIsa+iNKMJZ1Y3qYYhjGyokQFRPsItpkiWLMExQyjTmsAEJGq0VzVARJTG67ODZFpanSGpGPBTFILWF8hW6aQJyG+kaVtMmqQpiP4IkIbOUQrsKhUbM+igUUTrCms9i1cHqSRBFD1kHnQESoOjiZF0st0H5lsCqOfj9NtVFGxFaqAKM7chQvWvYOuk+wwJBQ2rAhBDEDdO02+kwKhTkdsDOL8CBP7+p0/yplJSyk3IuhCCbzSKlXJHBuBw+2a4G+ywlpgshRoG/jskDGQf6MX23G6gA7wsh/jfgstb6oU/8U9Ww16L13b17lytXrvDaa69RKBQ2fUwpDbNCCMHAQUH3XsHEBwmVu9CYM/LxVLfJWzTCFYMVa4xdaxxo3IKiWFqA0MZTBfy8hQ4N+0PYxvckrJqfF6IFg1TMsZ1US9gRmybuZARJaMz7hZu0cGmDYd5bOmq8gpm2kwiiIKBUKtFd6MXPeGhtbvedlIFIREMjU9q41wnTrBzbMjzlFpdaI2jGEbUwwms152aL1ufZRore43tIIai3uNcCgSsFpWaTLt9HCgNN1FvvUbPl6ieEGSssYZLaEaKjggTQQlCOIqOIbDVjS0hmqzW6035HeIOAuXodr2lhL/qIqk2iBAKBSowVQNwQ4KhW/iNEYYKKLQInJKzaiCVJqhuiskAoG+UlxnVPCGIZI2ML25dUZQMiSUSMU1CI0OQ96q6AKFLoyFA/A7dBPk7jFiDy6+RElmYUI9OKbB6Ki5LMMDiZGM+VpLoS0r4gmwUdmYYsLGMUJm2jhHV8qNw16TF+D9RmDNVQuvDmfwNj7236NH/qlSQJvu/T39/P6OgoWmtqtVrHW6her3fgk76+PtyWRcFWqtlsPksLRxsIgd/VWp9b/gUhhAf8V8AvAv8c+GAjB/vU1PIrcJIknDt3jiAIeO+99zZML1pdq2EWJy3Y80WbyqTi1jcSmkVNY0GTHjTSvWYLq7YcgV+AMEyYm50n5WbpGkszc6tIWDZpKOk+aJaNPavlCuyM8chOAnMRUHE79dt4hYQN09TdjIFdLO21/LgN3zqs3WvUcdOIa2rNCkEjorenF8ezTSajL42TX8tAyZYWOtIIt5Xd6EiQRvwStCxRwzhGCkHO8ygHAeVGg7zrkvc8LMuiGkWUopC0bXwt+rIZGnGMRlPwfaQQNJOEMDD+JznPI2w1btuyOtxoS0qyjku0DM1LVEI9ilACUtJCC+PShxBUw4iMY6TtuiJxpj3cwEPF0kApoSDSLTVqbKLIkiWJchKiRKOUIFYxSROwE0qqgTvlk7YtlKdQCy6QoFMKXXQRQiNyClnzsIQktAPCqsQVkkSEqMBcGBKt8FMSa97B9iRuCqxFBz/rUk8iHOGQlC1kzUEAzmyayhIkwsJuuJTmzLmcG4FSK1vayZqbqdIN8/fsiIHohCXI7YQ3/ib07N/Saf7Ua/XSsT1xZ7NZ9uzZg1KKYrHY4X9rrTvLzZ6enk3BJ5VKhXQ6/Th+jU2X1voW8CtCiC4hxH6t9RUhhA/ErTSbX2/9t6H6VDXsdtXrdY4dO8bIyAgHDx58pFup9XDx3Ijkxb8kmDmVULyuaS62LE67BHGrsQZxk8Vike5CLxau8RnxEpy88eBoLJlGrELdSStP9UqiRkJQNRCIlzUCmqip8XNGwBI1zGMpGaKFJmpRDIU0RlNCGtVjca4EQtA30GNSZmTLVD9RBnt3zTIySUy+ZRIZ18E4TEyIgVAdjrRtSVLCoRIEuJZFwfOohiGulAiMErEahqQsB1sahzwhjI9HxnWZbzToS6dNk9Z6xW2ptez9iRPFUhLg2Rbtj2CUKBw0YZxgORIXQalpGn8UKnTVQiw6RA2NjDShEqjAGC9pWxnZPpqkIUkCwFGoikVTh2Br4hC0VASx8UFvphtU69BLmiQVkYRglRwSERJbCfZ8CisOkfkEZz5NLAR2n0IuppBKkLghBJqF8w2EliQ9IY3bHlbTI9NvszjnYEkPkbPwslC+DXnHJz0ElRtmSex1gZs2y8T0kPn/OMA4G2ZBusZPxu82Ksa3/44JcP601sNYIlJKent76e3t5fnnnyeKIubm5rh79y5nzpwhlUp1xDvZbPaBn/lniYMthJAtqONt4L8QQvyS1nqq9bUhDMQ9s9HjfeoadhRFfPTRRxw6dIienp5HPp5lWURRtObXpCUYft2mZ7/Btsu3FcGSURuWG0Xq04rurkFsIU0YQaCMY2Zdk+4xQbhh1eQY+q1g36CkkI5seZBomlVFqsuYOoX1ludGtrWgjF3cFK1FpbEftVOCoB6zMF8inXcp9ObQymDbKlHo2HCxlUpa2Y4tG1Zh3PyEML4mcQvDDuIYjSBMzJ+ebWNpRdg0hkfz9Tpdntd5TRYadQqeR6w0vm0RC0mx0UBo7lnfum6Hay2FoBndg7E826YahkSh4XX70mI5yNWIImzXJYk0fsnDrtskgY0lIAoMFS8OzLGTCMKmohlHyMTCtSy0FoiKsS9NliRoiN0YIoPHK62wtU0lCWh4TYS20Q7gB0SVBKEFjWyNIEwo6BSVXEAsE7wwg84kxG6EqkiEBzsP9LB4ZwpXeOhMldBvUitH6LwmlbfJZgRRxSXbaxbZcUPg5BIcX+FmWwKYGJwcLF03/y+E4ViXbpiG1H8Q3vl7G2eCPKu1WaWj4ziMjIwwMjICmCY8NzfHhQsXqNVqdHV1dSZwb9n5CWbCflaMn5bh0seAN4C/IYT418AA8DeAU5gJ3NJaP1Rk8qlp2EopLly4QBiGfO5zn3skjGt5tTHsB5WXE+z7sk3xhmL2XMjU1RK2bTE41mV8iBtmkWf7AlEzU1ywZKTjwjae1GHFBO0qqVEtlkeqV5AkRhQjpGF6hDVFWBV4BQn1iKgJVmSk8HFTUVsKWKrNMTDch215qNhEiyWB+dNyTHM3j23WgUIaHFnHIBwBSmFLm1jFeLZtPESkRawVlWaAEgLHtim0bkObcULedc2krTVT1RoDmTSRUiaHMcH4h2gQQhIpI6oxIhyJWvb6LvcPacYxhsBy799kIAhmBflGFh1ZoARxQ5MgSGJNEregj4pDbEfEShNGYElF0FQoJcj5mrBigYLQiZBFHy00TSvEq6SRFiirQXVekPMl2k9QdQsRaZSMQUlEJLFdiXY0VsPGS9vUdANdsXGEICY2k3nTIZVO0zVgU7laJZ3KsRAvUrsZciNYwPUc3FGI5lPGrVELotCi1PJmz41C+Zb5fzeHsSioCdIDMHwYDv4faIVMf7rrUXnYmUyGTCbDrl27UEqxtLTU4X8rpejt7WVgYIDu7u5tlaULIf4J8CZwXGv932zxGEJrvSCE+ArwPwL/V+Ak8D8DvwWwkWYNn5KG3Wg0OH78eGebvFW8eq3ajHrS6q0ymTrBwEsvEU/liGtGMJMZEDRLCpRAhQ52l0IkGsuHqGa2/EhazBIjikGbJaO06Tj+hRWNXzCGUnFDIxMXy9Po2MSKNZIKjaDKQP8QNrZhgkQmEsxJGYm8sMBxW5askRHXCCxUrEGbRaihCiosYZFoI24hSdAYDLseR5SbgVkCKkUjCrGkhyUltSAk5dhorUi0aHGpTS02m4AmZRvuMhhxThjHZikJK6ZtgKVmk3oY0ld1cGs+omERRopQCUjMXYllQ6MZY1mSqKlRkUS5IapkE5EQ2woROQQqJogT4kjj+8aPJAmAdEQcgyw7JJYishO8xQzSEoSZBHvOR2lQmQh7KYUtLUKrhhd4+EuCRCjSXQ6128YmwOtXMO9TF5jJ3rVoTNhYkU1ut03uao7uQgo7rYlpUp4PCOMSbs7QQlNph2zOQzom8T43ZnjyQhqZeRLC7h+DF/7StpziQCtD8ynS3LZTOCOlpKenh56eHp577jmiKGJ+fp6pqSl+7ud+jsnJSQqFAmfPnuWll17a8u99/PhxgKzW+jNCiF8TQryltf5kM8doQyJCiD8L/JfAGeAOhtL3ey0ce8P1zDfsubk5zp49y8GDB+nr62N2dpY4jretaW+0Yd+5c4erV6/y5ttGkNNYVMycSahMKpJFsDxpnPEqLVl32kzaWkFQ0tieQPga2zXpMbZPy30PwtjwtIWtiWqGPeKmBaJojKb8HCyW5rBw6e8ewk1ZRIFCNYxcWdrGgb9tntRWPiJMsHCSKGwz/iJtEJa5S4uSGCElQWQk6qoFrTjS0Pfmq1XmZmeox4qbQUBPLkdTQG82ixCSxXqDgUwagWCpGaC0Ju97RK1lo2NZnQa92GiSdV1ju9oaqF1lIYsW2foYXtM3YQuBsTPVEWiVkITGJqCRxPjCBOMqDXFVItyEsCKwKj5xKiJqCCQ2gZ0QBAm+ZRNbCWiLRGiCTICVWMSRopmLyEmPZiDJ5ENCEkRDIvMJdkah5sBPS7QMIdKkrQK6q0YkI3qsPFFXQG5UICcT0mkLJy+oVk025fieFkNBSOJ6Grc3jZOF0t2Y+mKJhXKTRbeMqGZwPY/CqEtUMt4l0oE3fgF2fG5bTu9OPekAgbUe/3EpDx3HYXh4mOHhYX7/93+fX/u1X+M//+f/zH//3//3nD9/nn/wD/4BP/MzP7Pp43744YcAf9z669eAI8CmGjYdJ30s4H/VWv8HACHE/wP4/wgh/q7W+s5GD/bMNuzlIb3LE2raDXY7G/aDeJtKKc6dO0ez2VzBRkn1SHZ+VrB4VTFzKiEJzGLPzoeo2DEiGgycEjcVdsqYN2mMN3XcMEwQNw+WZRaNQrY8SBomQFjbMXgh87N1/FSaQm+GONSEdcPdtj3QSqMSgW0bTBdhcOs4TnA8ie0L7NDg2AjDAweJkKpj8uQ7DkGS4FoW9SgiiGOCoEm5WCRd6GIglSJbbxAHTYJajZlKBd/1cFI+6DSNOMJqeXnaQtDeCJgQg3sXw3ocEcYJIzqLXXOxAkmUQBQlBCEm5VsrksiYNom6Q4IiUpokEhRVA1+42EKiXI2qGyvYxA/RRQeJJvJD7EWzMJW9Cll2wRJEdoCsO0YOryJIJK7tshQ1SLuSpG6uJLYPdsMFHeK5FlbFwooETuKA1IhFFzvrIq2Q8h2QCz5hYiG7wJvNUbEMp7pZFGa6l2ZZWLoGYONmIZfLI5VDkg1JvCqzs0WEBdkBn0N/1WL8nTywvdPwVnxEtrOe5AUjnU7zmc98hl/8xV9EKRPBt5VaWloCaMeYlICXNvPzLSgkAdBa/8fWvzmA0Fr/ciuN/RXgTlsN+bBjPpMNOwgCjh8/TqFQuC+kdz3xzFbrQRN2s9nk6NGjDA0N8fLLL993ayWEoHe/RWFcMnM2oXQ7QdddlN8K2q0YZoidNjCHakEb0hI4WePEloSgpcD2IWpqQ+PLGIELVcnSbJWewTyOdInqxizKOPHRCigwrnxRA+xWXBgC7JZvdxyY4ANlaZLAUPpUrLBsCZb5EGulcYQkRuPbDpVymVK1xuDAAFHLt8OzbbTwGcpkKDabiERhxzFnb9+mx3Xw0mlc1yVw7p1SHSGMhkxs4wYOTuBAJEmUJIiN90lYB2knSGWjMEk+OpYoK0KXbBIladoBNCWBnbAUB/S4KUIZElsaEUsafhOZSKyGTdMKqTshznwa1xYkfmxsTLVGdMU4s2nQGq9XIhsOquEBEcpT2HNpc7chFWrexSp5kEoQSiDmUlhKke6VlK776CxkuswCOSxBtkvidRtYw+s2iUACw63OjRvIozGdYDlgS0Eq61G965PVkO5N2Pdfz7Akpnn//RK5XK5jebodku6nPWHDk1MeLqf1SSm3TPFr6Tryrb/mgaXN/HxL4fjTQAy8r7Wuaq2jZV//74QQBSFEF+aC8NB65hr24uIip06d4sUXX2Rw8H4e02Y9sR9W6y0dFxYWOH36NC+//HLH6Ga9sn3B6Js2XTslF/5EmfzGmvHETmLjCwEmsktYJlwAZQQj0jYxYio2/GthmyZWWarTjGr0FAaQykM44KZMAw4brWMp0+SlbUQySpllpi1bid6xNjat2kziBia5dxFXSrUmb4ElTbDt/MICKknYOTxMM0moBAGJMsZQzShmodEg47oEiSKTy9LtOAxm0lTqdZJmk7lqDcdxTMiuJemPMziBg6UkYWiEPnEAAk0YaANvBBrVFCgZomKwLYlOIGoKpJMQ1jVW1Sf2QuKaJAyhkglJIuOrokiI4gRXClQqotmIUSiWUhV67BQilgTZBgpNQaepZBqEMmRI9JA4Cc103QhTlMTt0bieQkxL8qOS2VSEiI1fuegLEEJhixz+kCLbBV7oIxIMjJUR1GdbbA/bvNQdmfmIofc1ix52XlIYEa2pG/pfgsO/ZOHlRoARtNaUy2VmZ2c5duwYcRzT19fHwMAAPT09W4IWHoePyLNa9Xp9zd6x2Tpy5AjAF4F/B/wI8P/bwmE+A/QAu4QQEbCAUTg2gS7gx4BJ4P8NLD7sYM9Mw9Zac+3aNaampnjnnXfWvSpu1hP7YbX6eFprrl+/ztTUFIcPH96UYirTLym8XkYtZBFLfkdk4+WMR4jWJqxASIiaCh2aCczJgrA0KjZUwHJ9EZHYZJwClm18uqOmIqiZdHYpDB9bOsZmVSWKqGES2qVlRDXCEtieMYkSWhp70Ni49dmOJI4M9xoLLGka+tzMjHFby+aIVGJSyC0Lz7EpNUND+bMkvm0jtDbBsFIQa00+myWOEqxQkkscUg0XAuPn0UgMmB83Wx4pwiKKAA2qZuAbLTD4NTE0Fa6wwNYkTYsoUUR+jFXyCJKYyFM05jwcKdC5BLXkYCtBnIlxaw461CiRIEJJ4pjXQGkQoY2dssBJsKo2nm+hpSaqCKQ2LoGOLdFLlsljFKBnXKSyaGqgbmOXbcKajZu3qNwGlVhYjuHnt21RU333KHt2xmQthmVBZhDqbpNsj2GM5Mah93l49b8203i7hBAUCgUKhUInEaa9VDt37hy+73em741Kur+fGna1WmXPnj2PfJzXX38doCmE+BZwUmv98RYO0wf8E+B/wHCiypgGngfOAf+z1vqjjR7smWjYURRx4sQJUqkU77333gNPrMcJicRxzMmTJ3Fdl3fffXdLJ7hlSdzhgJFDLrPnY0oTirCm8AoGAgmqCoFZELppYYIEtBF8CCemOFvC91PketKU6vOAb5gmnmWEM6HCcgRuSqKUImqaKdn26fhj255xwYtD1QodNxFoOjG0PwAdaxQSgabeCFlYmKOvv5d0OoVC42qJKyW+ZZEoRVVGWJbNVKWKIyWuZSMTzaCbxlEOSUOTTlySmHvpLIFhpohAEjQTYiuGRBKpGJFYWImFkjGqboOAqmxiBRbS0swnDbrsNFomRMQQC8J0k0otxg1sQjekEsX0l/IoFRO7Cc6Cj5CSulXDL6eRCIJ8jNvwcEKbxI7x0jb+YpYgikkPW3gTaSwhiVIBMrJI5hwsS0AmoXIbLGXhdhmYSTclXk6QGwenlMbuxbzuCaAM20PaRlIuesxdjpuG+qwgDsySOVqyqZRsLAue/0vw4n8JD+u3tm0zNDTE0NAQwH2S7nagQF9f37q7ne+3hr1d8WBbpfItqxRmyu4F/rrW+syjHOypN+ylpSVOnjzJgQMHOiT5B9V2QyLtpWO1WuXYsWPs2bOH8fHxRz6ekxKMvuHQtSth4UpCfcFkFtqeMKG72jQBoY2CMahGlGfKFHqzZLs8AxtEPtprmUMFBt5w0wZzDhuqE0igkgStJdISSNkK7FUmf1Jaxp1DtNLAdSKIoxaeqQWVpSqVapn+/kEcyyEONFJIpNC4SBxpkUoXEKGRtQ+kfCwktrJMwHDUaswKoyRMNFGiTNRWAjKWaBKEspCJpKljs1RUEUGjSd5KodwI3bSxqy6RG2HVbUToUPEDbGUZNaFWBJEyxkt2TBQrqnFMd1dAsxmjY0GQaxAjoSrR+YjISqApsfyYKB2hAoHrWFAICcIAkRTQ3RG2byxuhQO5AaPMtKalgTFqMa52cF1IdwnyOYtgiZbKEqozogV5GU710tXWeZAyniDFSdON0/0mpEJFgtSI5tDPbZ0Jkslk2L17N7t3717hiHflyhWklJ3pe3ke45OO6FpeT9ru9BlLTP8fgL8AHMSIZi4Ad4EZYBa4shHTp3Y9tYattebWrVvcvn2bN998c8NE9+2GRKSUneXiVg2kVh9v+QUl02uR7pGU7yjmLsWoSGF7JhpMJQZzrizVaYRlBsb7EMomrJoUG+wEFTooz4hqtIKo0Upa8U2mpCETGJ8TLYSRtQvj9mek7BoiCyViY17Uyi7UiWa+uGgSs3sGkNohaZr4Lq0NdU5gmq5SNgWd5W6ljFQWidZE2kA4osXgQEEjibES0xSSWJgNfRjjhC7aVsaXOrEQJMSRINaCUhyQUQ5BHNK0YuyyT6A1kRvjFD0CNFZOYy15EEcoL8atuTSjCGVFNCoaHVsoYdJtkkhCYgQoou5RjUJiJ4K6hVACW1tIbeNWIXBMurw1l8Fpxuh0bLjdNYG15MMQJHc8lG0RFmDALhBMCZq0QnAnTQiu7RuTpnbOopttMXYQ5He0/K0bLUZQM+a1v5kw9tb2fPSWO+KBWdjPzs5y7do1yuUy+XyegYEBHMd5ajzsJ81QeZak6VrrU8ApIUQWOArsxeDhWaAK/CxQ2+jxnkrDjuOYU6dOIaXkvffe29SVfzshEaUUFy9e3Fb15FpLTCEEhXGL3IikeCuheD022ZFaUwkW0SJhYGgQ1TS2mggjnEFJRCo29qk1M6XYvvHOFsIk3Qg0lm3MoYTEwCHSTORaGeMgpEYlrXR2ZabuuYVZHMemt2cAHWuiFk9WqdZ0bgl0rIljgRCCVOKjGjXqcYxUEokiiTDeGlrjxA5KCyId42ITJAk6FAglaYgQXbRxHEliJ4jQJoqbxgI1ETRkBMJCaIu63yCoJzhNh9ANWaoHDJUKJEJRtwO8UhpLCqqySrqcIbAFVj7BqnjoRBA5IYSS1EIOhSJxmui5NJYQxNkQWXewlzxkYgKX7ZtpbNdCdDfRJYmILLw8SDsibKXe5woCKzbNzsliBEuBkZArBbZjwimSyMAelTsGjwfIj0HxsjkP0oOw469MMHCo+5HPs/XK87wVeYylUqnTwJvNJo7jdJaXT6qJfj8npi+j6/23qydpIcRuoL6Z4z3xhl0ulzlx4gS7d+9mx44dm/75B3l/bKaCIODYsWP09vaSblHStqOklOvyuqUl6N1jUxizmL3U4PaFeTL5FCk3i+VIdKKI6kb96GYF9QhU0yIWGscTCEuDbOU92iapPQlbIQtuy241Nn7WCHDTxriJhkBHxthfWDF3J2fpKhTIZLOoWCGQJE1FRIzt2KhIE6kEaVmoUKASjWVbZKVPpVlFK40vLGKVIAOJjG0iN0FHklgLGkmIg6HoEUEUCRpWA7/hkRYuDTeEmoMDhE7IQjmmK2WbBhsJk8guE3QC1TiglqsilEQFika2hoWFqAka6TrCt0klHkk6IrQiwpomscHpTgijBMoC3R0SiAQZW2T7JJYVoSoBwtLIvhA/K2mUJCqb4PeCZYFfclEJFLIejWnDqRYWpHth6aZ5P/1uo1SsVs3fc6NQnjAXaDdjpu64bixU8zvg9f8zHD0bPLFJVwhBV1dX57/Z2Vm6u7u5e/cuZ8+eJZVKrVhePq560g37WYJE2tzqltqx/caL1pdubPZ4T7RhT0xMcO3aNV5//XXy+fzDf2CNsixry0T4dhWLRU6dOsULL7zA4OAgU1NTj3S85fUwIQ5AuVrk0vwpXvyhg4hygcpkYiLK7JbQpmUapSMLvATbMzSxODBN3/aMZ3aCQDqtjMREE0W6hWtboBU6glgZ6EKphLAZsri0QG9fH77nmSTuyHhUS0dCIAibCsuVEFvEgfFI0YkgUZp+P8vdhRpaa5aCgJRwiXSCjMAOLRoyREcCISQBMbY2EvVaEiJjiwYhwoZoSYJWRE6MLHnoCOpWTKqeQkYxSaqBV0kRCwW6QVx18ISFUg1IQCYCIk1sK3JRxqTK2AGi5OCiiXWIGznomo3ViEhlbJozLpa2SPpBLHg4ZWgkEuFYJHc97EgS+w2CBdNk7aaHOygoT5j3zMkYi4GwCn4veAWI6uD5dDjXcUOQHTIwku1Bdco09LF34a2/Y37+aUnElVLYtt1RBIJpbLOzs5w9e5ZGo0F3dzeDg4P09fVtKLFpo/WkG3a9Xn9mJuzltUwY8+xnOi4sLDAzM/NI3tXwaJDIctz8rbfeeixX4QdN2AC3bt3i1q1b96iLI9Czy6Z0J6YykxBUTNN10gICE2SQhEbibhq1wb0t2+DgaJNaA63cR62h5QOtlMZ2BCSCZj2kXq0zMDiIZdkkQWsR5UiSwIT82o5ZRIZV04C0ki1nPEPF01rjYFGpR2gtaegYEVhoEhAKL/SIrYhYK0QiiaUiUIkJJhYxOpGUGyGWDzqUyFiwZFcJQ42s+NheRFNHuOUU2oKSaJKpZlGWJOyKcRdTCC3IdVmUaopU1SZxI3RGY88bMyq3VxHO+sYYyw5JZIJf7KKRNNC9Ic1FkAhELsQtaKySRboPlBURFDV+F4heyOBBbKZmA+YbvxchDD6+dN3g19IxcXHllrueWzD/Xps07/dzfwFe/iv3mCBPi62x1uOu9qNeXFxkZmaGS5cuYds2/f39DA4Oks/nH+ki86RFO41G44nlRz7pemINu23W8qjTxVZZIkmSrMh8fFxX/PWEOEopzpw5QxzH9z2+7Qt69zl07bQpT8aU7sYgMPmLscLOGutWlEC6LVqeNh7bCI3jSWO2pE2ziCKNbUtUpIkVlCtlwjhkbHTcuN9FCZYtUYFxwrM9I8IJygqrxfOOaiAtBRgzIm1rhBYMpwpMLU3iaQcXm9CKsQMbFQlKToNU3cfSEHkRQUliWy7KbiICB9AUozo9La/Q0IppNhRYsCTqgI+Simamjo1NWEsIUgFloD9ME6cDYithNN1P7C0R+CF7+wZZqjZoFBpUoib5pkszHeMOZNCRQDUVuX7BdKlGVvukesB1EkRJoyKDsftpQe22j1WzafomnKKxaF7nzKAJEVARICE7BOWb5n3zusyFLQlEK1WmZTalzQS+98/A3h9beR48zQn7QU1z9fKy2WwyNzfH1atXO8vLwcHBNe1MH1ZJkmzrxL6R+l6lMD6xV3G7TtKtsERqtRrHjh1jx44d7Ny5c02J+XZNPmtN2M1mk2PHjjE0NMSePXvWfS0sR9C906Fr3Ka+oKgFiqgiiQKN47eem4IoMJQ+J2UMm5Q2E3gUmSBfHZlYQSE109PT2I5D2skZTxNHopqSWIHlSKSEsGzUl7YjTX6l1Fi2IKxgQnxdQdIwXto5xydHinIUGMMpJQlkTALIwCKyI8JQoRYl2orBsnFKKZSlqagGqVoG7VpG7l3ySMWSqlvHrfsktkQ7xnkPJYlDk5zuRR6279C0m9gVD89zkQhkzcXzPZwgoaES0sUCPZkMRT1D/aai2oywhU3sRzgzGaTjoPsFat5DlhMiJXE8aE44uErSN2TelzgAv8s05KgG6YGWclGACoxqUQgjOQ9LJlQ5MwjFqy3WSAoO/99g6I373+On1bA3O+X6vr/m8vKTTz5BKbVCefmw4z7JCftZS0zf7nrqPOzN1mYb9szMDOfPn+fVV1+lu3vt7Xz7mNtxUq3GsIvFIidPntyQxL1dQgoy/RbZIKBeadKTz1OdVSYXEm242Nq43qkEVGj43SqGRJjb9CgKmZmdpTvXi9aSMK6hI4ugYbBy45utO1zuqK5JAiN5j2qaoGEuCGHFOAh6uVZeotb0pdPUmwnlRoxrSxws0JqQCBWbOLHISfAi19wlpGNoSOKGIHYCosSiq5yhrgOadkS6bCbumheQr2RAC4J0E6fpYiUW0tXYloW/mAE0uS6b7K0MYaJIZ22WmhZW2UfJBH9A4V7z6R/vIuM2mZ8IsWJBmGqAX2NxXmMVJJbfINWdodGQdPVIhCWJm46ZymPDBindaIlfsgYVaS6Z9yczBLU5E4qrgfw4VO+akIHMELz+C9C165FPpW2tRxlIli8vDxw40LEzbafBtMN011tePmkMu/2cvxfrU9ewbdveECTSdvtbWFjg3XfffeBt3EYWhRut5RP2fXj1Fo6lZUJh1KEwajw3GktGhNMsKVSicHxJHBt2iO0a35BatU5pvkpPYRA/7dCoBiQNC1LGNzsJzC2+7UmSwDRqyxFG0NJoBQ0LiKpmSpeuoLFg+MSWK+gO8yxGEZW4QlwXKEvRkBEikFgWhDrBio1XdQOFGzs0RUTdCRGJIEaRzjWpVAKEFlRSVVxpk8SCIBOQiAQncYhTTaqiTo+dxk9JllQVgcbGIu4KqcQNMpkRrKYilg0inWDrHEkmwu+FYMnCRpLPFiAskap0oWyfpl0mKkqmp+tI1yPZm1C/bT4Klm+olaWb5j1ID7S8YCRkBsDOtih9rWnccg3/2skaccyR/7vBtJ+1Ukptm8PlcjvT5WG6Z86codlsrlBetndOT6phtwO1v1frU9ewNzJhh2HI8ePHyefzHDly5KFv4EZSZzZabeHM6dOnCcOQd999d8v4nZRyxS2e7Qlygza5QXNBiuqaoKIIqib3MQk1xekqYRjS19+HtCyDqQrQMgIhUREI2zj4JWHra8oEy6KNo5xqXQ+dFIR1I9jxCoKw2pq2M4KhoEC1klARDazAxotSBLYxZMpFWWIZEwmFVfWwHUmVMqmGT4Km6jaIKxIndIjsCB1JCk6GkgpIpMKuejiWjXSbpBbTZPwMruMiazZe4FG1wBGS3EKBqgcSC8oOfjNFEtgIIajcEsSJhaVMo3AWMliuRe9uHzFtEadKdHV7aCti5m4VLRTpnIfrp7GkT25EIBwIy629QN005OLlVqP2zeTdDswdfgve+UUjonkW63EtO1eH6SZJ0lFetpeXnufh+/4TgYMajcazlJi+7fWpa9gPY4mUSiVOnDixYak7bK96Mo5jisUi/f39jxwQ/CDGieH6CtyMJIeZLE6ePIk95vLSnudRoSQJTSJNtdykGkX4eRNq0OZuCwFoaRq0MFN2O1fQcgU6gVSfQtgaW1oI20zZQgiGLZuLF5a4NDNNJGICnZAIRSIUWc9wsG1H0FAxQ9kME2VjK9yME7TW9KRSxMr4qiw16vy55w4wv1gkTmIEgqFcloVKlUsLC/zUgf3sGx3hu8fP0ZdK84XPDHDs/UmuzRf5hS+N8OGNKlcvT5CxU7ywO83xj2d5+bVdNGqCS6cXGdk5xIlUET+bJeXk6LIltckYP2cTzWfJAX6vojQZMFsNCaMameEEvZjHc11s38LvgsodY4ubavmEAOR3Gqz64F8xF7tntZ4UO8WyrE7WIpgGeuHCBebn53n//fcpFAod+GS7tA/L61nKc3wc9alr2A+ahicmJrh+/fqmpO6wfQ27jVf7vs++ffse+XjtZejDqtFocPToUcbHx9m1a9d9X5clwXy8xNDB3Zt+DkmSEAQBvu/c94H/2eHnOP3VCSp1Y10qECbP0WndGUgLoSDduhWXQnTuGPrSaWZrNUTLqH9XIc9iGHK7uIQtBXnfp9aCvvrSaQZzOaQUpD0H25ekCxI71GQGBD2xTbBUo7dgM/aqTVIqM/4ZgdYWVrbEiz8k+MPROQYGBO8ckCgFJ0/NsGunT09PyoQl1yRhOUVUTRHUYGmyztydEsWpKkFToxrduP1ZPN+nPm2SgISAV/4q7P9zm35Zn3g9LS+RVCpFoVCgr6+P8fFxlpaWmJ2d5eOPP0Yp1UlC7+7u3pYLyrMkS38c9alr2GtNrG3KXBRFvPfee5uGILYDw759+zY3btzgzTffbGfBPXI9jNMN9y4S7Qi1tUosa5SbKaUUSZIQxzHlchkpJY7jYNs2lmWRcV1+4cg7/Mo3vm3sWPW9n2u/TwPZLM/393FpYaHTnD3b5sWBfuZu3FPl7irkudtscm1hESkE3SmfRqth92ZSDGbSCAF+KyAh7br4rfc57TjESpN2HVKOg9tqTEIIcq0priuVopDyO6+rJWXnOVquIOVCatlOepQMYCa1KIqYm5tjdvYSxWKR4UyWnvQgPYUBesafUQxkVS1/T57GY1uWhRCC7u5uuru7O1mMc3NzTExMcPr0aWPt25q+txo68IMJ+xmvRqPBsWPHGB4efiBl7kH1KBP26ggx27a3jVq0GsNeXe07ioctNTc6qS8vpRRxHGNZFvl83li5RhFRFBEERl5tWRbj2Sx/+93D/L8++ph6K78x1hpHCBSaL+zZRZfnYwnZEY/0Z9LsKBSMWyEmVqzb9xnJZomU8enu8lOd4/Wm0ri2bZwD2w3btvFaDTvl2MRKkbYdHMvCX7Zcy7WWzYOZDCO5e+razbxHjuMwMjLCyIgJF6hUKszMzHDh9jHi63FHYLJdU+LjqKfp1rfe0nH169pWXp46dYogCDpJ6L29vRsewp4lWfrjqE91w56fn+fMmTO88sor9Pb2bvk4W23YQRBw9OhRBgYG1owQe9Rab8LWWnP+/HlqtdqG7ige1vhXH7s9Wdu23WlAUko8z8PzvM7X28173Pf4hddf5Z98cpym1sRK4UhJynb40b17KDYaKzyfx/J5dnYVWhazAl+aD/NQLkekFJ7SdKc8GrHxjOlNmyWSa1krpur2/2dcl1grUq75e9a917DzvmnYz/f3s6PrnhPjVhdgQgjy+Tz5fJ79+/d3KG7tKTGbzXamxGdp+fU0/bA3QpkVQpDL5cjlcuzdu5ckSVhYWGB2dpaLFy92TKsGBgbI5XLrvnc/gESewdJac/XqVaanp1cE9G61tsISaft4v/jiiwwMDDzS469Xa03GURRx7Ngxurq6eOuttzbUdDYKiWitSZKkQwFb79hSyg48AqYZ7PN9/t6Rd/iV73xAJYrwLYvP7RxHAL1pExIQt36X3d3djORzKK2xEKRt07AHMmmiJCGx7RUTdner8bm2Rar1mGnHwWtNbZ5tEyeqg5Vn3XsUzp7Wz+7tvZ9rtx0X2NUUt0qlwuzsLMePHyeOn53p+2k37K3AlO0GDeZOus08qVardHV1MTAwQH9//4rlZbVa/UHDfpYqiiIajQb1en3LqTCra7MT9laXm5ut1RN2O2Rh3759jI6Obvg4G2nYWusOv32z3slSSnzfZ//wEP/Tj32JX/vkE6ZKVX54fIxyuYxlWTiWJGy9xgd6e/FsG1saSX3GspYxRzSJVhR8j1oUIgRkWo3YtSxSrQ++79yDRDzLIlYmQBhWTthjhbVNxh6HIm759L1v3741p+8wDJ8K9expN+xHfexUKsXOnTvZuXMnWmuKxSKzs7Ncv34dgP7+fsrlMuVyeVs/k0KI/yPwS8AU8LHW+he37eBbqE9Vw65UKhw7dgzf93nhhRe27QTcaMNWSnH+/HkajcaWlpubreUNe25ujrNnz/Laa6/R1dW15eOsVW28WkrZWQ49rNaDFPqyGf67L3yeC3NzFAqFDnSSsmzKyrgsjqV9oijClpJmnJBrQRlCCCwJsVLkPY9aGOHIe8/Htax7Tdq28FqTuWNZCClIt76WWyaSGnmAa9vjXsKtNX1/8MEHnDhxgiiK6OvrY3Bw8Il4Uz/thr2d+LkQouNN9PzzzxOGIXNzc/zDf/gP+fa3v01vby/d3d18+ctfZmxsbDse8n/SWv/6dhzoUevZ3JCsUZOTkxw7dozXX3+ddDr9WGLCHlRBEPDhhx/iui5vvvnmEzGzaTfa69evc+nSJY4cObLpZg0PnrCXLxdt295QE4sDzeXf0ZRurj+lvtDi4VqWhe/7DOZzuLZNtsXuqNfr2EKQqITMsguKJe/BHDnPxVnWZFY0bMvqsEHATNztZWPOu3eLbK3TpJ6050Qbo/U8j3fffZf33nuPnp4e7t69yze+8Q0+/vhjbt26RaPReCyP/73UsFeX67qMjo7yb/7Nv+Hnf/7n+Yt/8S9SKpX4q3/1r/Ld7353Ox7i/yKE+KYQ4ovbcbBHqWd+wlZKceHCBarVasea9XEnp6+uthin7Z/9JKtcLuP7PkeOHNnySb9ew06S5L7l4sNKJZqbf6ypzxlDpI3WaC4HWlNsNjtb/ELKZ6HewIrCe9RBMGnqSpFxXZxlv/PypaNvO7jWveecth381sSdczfnJvekanlU1nJv6uXY9+Oavp+0xenqx35SDJVarcbrr7/Oz/zMz/B3/s7f2Y5D/g7wrzAhul8VQryptd6+5rPJeqYbdtvlrr+/n7fffrsz/W13cvqDlo537tzh2rVrm8Krt8P9r81AkVLy6quvbqticqPLxdWltebOtzXlO5qdn5fkx9bgxCeaW3+i6XtRkBu99/Wfem4/5+bmuTg33/k3WylUHPPmS6/R19dHFEX4tkWz1bxt28ZtPXcpJY4lOw3cs208697pm3HvsUae71+bj766njQveT0YaTX2Hccxc3NzHXOlTCbD4ODgIzFPnuaE/SQphVtdOk5PT/OzP/uzK/5taGgIrfVS669zQojLwCAw+ajPc6v1zDbsxcVFTp06xUsvvXQfC+NxJKevbtjtyb5Wq/Huu+9uyjinDbFs9QNSKpU4fvw4zz//PFeuXHnkxrJ8wn6U5eLsaZi/oBl6TdD7/P0/p7Xm7geayqSm54BY8e/WvM97Y2O80Uo7uX79OnG9zmBXgdGe7g5tMOt5REqTzWaJogjPkh3RjtDgYGiHrmUaeLsyjkuqtXT0NgBXPQ0bzo2KV1ZP321+8vLpu81P3vCd0fcwJLK8ttqwh4aGeP/99+/7dyFEXmtdFkKkgP3A3CM/yUeoZ65ha625efMmd+7cWVcQ8rghkTAMOXbsGD09PRumzi2vjSgU16vJyUkuX77cScS5dOnSlo6zvNoNW2tNFEWbWi62a+m6ZvJDTfdewfDba//cwkVN8Zpm4BVBYee97yle0UyfUIweluR32Bx7/xLKr/HSnj1cLxYZzN4TOvi2Q84zcVa2bZNPpcjn80RRhAXEYUilUiEIArRSnUaU811S7sYvqk/Dl3orj7manxzHMfPz80xOTnL27NmOOnBwcPCB0/fTbNhP8rV+DDzsvy2E+DHMvu9/1Fo/eqDsI9Qz1bDbaeqWZfHuu++ue1XeqMXqRmv50rGNVz///PMMDQ1t6Xhb4XW37WAXFxcfOUZtdbUbdhRFWJa16WmnPqe5+Sea9ADs/LxY88NXndZMfaLJjwsGX7339fqCZuaUJjcqSI8kfPAnZ+D2CHte34cOFqmqBKvikEhjPuXbNo689/MZ1+1M32nfpyufJ51OU4kTUKpDGyzYDtYmp+an0bAftWnats3Q0FD7dn1T0/f3su1ou7Y7MV1r/cvAL2/bAR+xnpmG3U6FaXMtH1TbPWG3G+zdu3e5evUqb7zxxiO96ZudsOM45sSJE6RSKQ4fPrztH6w2Xn3u3LlOzNNGLwhhRXPtKxonBXu+bEJ/7/uequb2NxRuHsZ/6F5DjwPN3e8q7BT0vBLx3W9/Qm7xBbrGu0j1Qs+5Lt4ULpPHYty8pnu3jZtYuMsm5ewyUYQtpZGoOw6FTIZcOkM+nycMQ/Z3d1Gr1RBCYLe+50HL1GcZEtloPWz6TqfTDA4OPvFF+dOsH0jTn0BNT09z8eJFXn311Q3R1h5Hw15aWiJJkk3j1WvVZsyk6vU6R48eZdeuXezYseORHnd1LV8ufu5zn6NUKjE9Pc2VK1dwHIfBwUGGhobWPcGTUHPtDzUqhn0/JUww8KpSsebW+xqVwJ4vSKxlmZOTHyviJvS+XeOjo8cYFa8h/CwDL0nmLiikDV3CR8UaLyepTCfYix6WI5g9l+AXDLdaK5PebkmJ3WrAlhQ4LcWl7/u8PDZqsnKThDAMaTabnWWX4zgddtHy+jRAIpup1dN3rVZjZmaGEydOUK1WOXfu3Kax709bPauJ6dtVT7Vha625dMk4oL377rsb9se1bZswDLflOYRhyMmTJ9Far2CiPEptdMJeWFjg9OnTHDp0iJ6e7Y0pWWu52BYbgDmxZ2ZmOikhbQl1m0amlebm1zXNIuz9cUGqZ50l44eaxoJm1w9L/MK971m4qKlOarzdJc7fOMPzw29RueLT+7ygPKlJYo3lahMM0AeWB42ipsfOYGU05TsKaUs82+L2NxVdu4WZsFtN15YSy7r3eG2+9XLJfFu0E0URzWazg927rvtUJuztgEQ2WsuDBfbu3cuf/umf0tvby9TU1Irp+1Gc8Z7FCoLgsfhsPyv11Bp2e7HX1dW1aRhgu1gi5XKZ48ePs3//fm7cuLFt089GMOx2fNjhw4e3Xaa8keViOp1m9+7d7N69u3Mr3aaR5XI5rLs7iO90sfsLDvnxdZaMl8yScfCQWPE99TnN3FlN4M+zEFzljYNHmP7QIt0nkC7U5xVOCqIGuDnI9Euqc4qkCaP5LJUwxE4JevZK+KZNs6qxUxInEp3GHJclMnwwFt/G633f7wiEoiiiUqlQrVZpNBpIKXFbOPnjrqcVwAumga81fbed8ZbzvreT0fGkbV2f5EXxadRTadht46StLva2AxJpszHeeOMNstksV69efaTjLa8HTdjL5e2PEh+2Xi1XLsplns8PqtW30jc/rnL1bBPVf4nmwhKDlw0Oms/nO8erzmimPjZLxoFD9x4jDjR3PkiYL0/jvTjNkTcPc/e7AmFpeg8Ipk8qLBfiWIGEdJ80EWc1gXTg5e5Bzt9epPc5EzScLNhkxgSZAYGoCOzW48+c0JTKEr1/Y02w3ZgBTpw4we7du0mn0yum7zb23X7ttruepif18lo9fcdxzMLCAlNTU5w7d45UKtVhnjzq9P0kKX3f64np8BQa9q1bt7h58+YjGSc9inBGa82FCxeoVCor2Bjb+Wavh2Evpwu+9NJLj2W5uFnl4uqq3IGlUxn2vJ5hz5f7CKOQ2dlZrly5QqVSoaenh97cEKWPe3FzYsWSUWvNnQ9ibl+fpP9Ik1fefoP5C5pmSTHypmTxqgI0wtKomiDVp3HTgvJdhY4FqX4T9rtvoIvskGD+gsZTNn0vtnBrIXAsi8aCpjGv2fvC+jaba1W9XueTTz7hueeeWzEoLPf6rtfraK07zdtx7k/a2Wo9zQn7QWXb9orlZLVavW/6bmPfm22+TzoxXYi1WUzfK/VEG3YURZRKpUc2TtoqJNK2Ji0UCtuGV69Va03YbeOq5557juGWeGS7avly8VGadaOoufHHGr8bdn1RIKTA8zzGx8cZHx9HKcXC/CIXfrdBaeYm3YeXcO/2MjQ0RCqVYvpMyMVPJhl/x+Wld56jPqcpXlV07RREdU2zpHGzEFbBzWlyQxbVWUXcENi++aDpGIZedEkCWLqu6O53O9i4bH0YF68qpC3Y/8LGl0ttMdJa+4LVXt/tBh4EQQc2aTfvRzlvn9bt+mYvFMun7yRJmJ+fZ3p6mvPnz296+n6SKse2gdn3cj3Rhu04Dq+88sojH2crkEgbr95MOO9WazWGPTMzw4ULF3j99dfJ59e2+3xQPegD9yjKxeUVNzTXv6KRNuz9MdFheywvKSXh9R66XM0rf1ni9NeZnp7mxIkT1OcU9XN9jDyf54UfGiIJNdMnEpysIDcumDqqcDOCODShvuleSdQwKewIQbpfUL4N+TGBlxPMnDaRY4M77y2QBIIk0JQnNF27BNYaFMO1am5ujnPnzvHmG29BNUOzqPG7H+z1bds2qVQKpRRhaLxOliftbGX6flqQyKP4iFiWdd/0vTwV5mHTdxueexJVr9efqdCIx1HPBK1vs7VZSKSNV2+1YW622pCI1ppr164xMzPDkSNH8LzNmxK102LW+qC3l4tt7vFWm4GKNde/ahgb+/+MwM2tfZzFq5qFS5r+lwRduwSQYe/evWRTBY59NE//SDdifIr337+EOz+GH/Wy/0cyzF+wDAyiNUloUsdTvZLijZgkEvg5SVQ1iew9+yRRXVO6rSnsFAyP3ksSElJQuq3RiaZr78Ya0J07d7h+/TqHDx/Gcz3O/SfDOBk7srHXqk0bhHsZl8un7wfRBlfX04JEtnOyb0/fe/bs6UzfMzMznD9/3rgytpgnbarok5ywK5XK93R4AXxKG/ZGIRGtNRcvXuzAMNupHnxQSSk7YhjLsjhy5MiWPzBteGX1z2/Fw3qt0loz8S1NdUqz+0ckmcG1j9NY0Nz9UJEdEgy9fu97bt+e4OJXy4wP7WXflzxSvYOUJhJuzjVQg3Mc/9ZtWMrTNZjCb+RIddnkR22q0wYKsWzwewTFa4q+5ywsVzB3PkEAPfsljn/vIieApZsav0uQWmdCXl5Xr15lbm6OI0eOdN777LCgMqFRb2uktbnXbK2knTAMO4vLh4l2nhYk8ric+lZP323mSZsq2tvbSyqVemIXqe/1tBn4FDfsh03YURRx/Phx8vk877zzziOZ8m+2kiThxo0b7Nu3j927dz/SsdaKCWsnmbfTyx+lZk8Zet7wm4LufWv/7nGgufW+wvZgx2cNtt3m0M9djBjJPMfwazapXoNVz53RdI+m6H9xJ5NHFWIgolyssVQvMh+VKWqXVNCHI9Jk+iW1GY2TNtTAqG4gj8IugZNa+XziBjQXNYOvPrj5aK05d+4cYRjyzjvvrGhWPfsEN29rlq5Bz4FHeuk603ebNtgW7TQaDWN4tWr6floT9pPyEclkMuzZs6czfS8sLHDjxg2KxSKNRqODfT8uJWK1Wv2e4pSvVZ/Khv0wYUp7wbcZvLp9EXhUmt3S0hLXr19nYGDgkZs1rPxdV9uiPuqHsHRTM/mRMXQaemN9jHziWwYu2ftjEjslSJKEkydPIsMUffELZEYFPc+ZJj59Uhns+RXBzGmFdDTSssimcvQO5+jZN8TdM1VKS3Vq9TncRoRXH2TXO1mkZTN3NQFhoJHV1Vg0ePd6vHAwF7MTJ06QTqd57bXX7muQuTHIDAimjikyQxIvv33c+4eJdtqN/EkbMT2NxPR2JmOSJHR1dTE2Nnbf9D04OLgl5sl69YMJ+xmtB00pU1NTXLp0adN49Wbk5OtV24uk7Wm8HdXGsNvLRa31Iy0X21WfN0rGdP/6hk4As6c1lbua0XfMYjAMQz755BOGBobh6k4SB0beNnzv4jVFfV4zdEhSndaENU2qR9AsaewU5EdsgqLGs9P09WTZ/Qos3AypqTIXJo6RXNU4E/sYPJDG9u9ngDQWINXLmhJ5MHdVH3/8MSMjI+teLIUQjL0HV/9Ac/2ril0/LNdUcT5qrRbtNBoNbt26xcjISMfr27btJyLaeZr87zYcs9b0vRz73o7p+3s9MR0+pQ17rdqqzL1djyLGaWPl5XKZd999l4WFBZaWlrZ0rNXVZpy0l4vb0ayjuub6H2osb31DJ4DKpHHa69pjJuharcYnn3zC888/j74zQLGs2PE5C9sXBBXN/AVFdlDgFmD+Y8PEiOoKhMYvSLw8zF80nGvLB2lJLO3y/HuDpHuHmTodcHe6xqK+zu33i3R1dTE0NER/fz86skiqgtw6sE2j0eDjjz9m//79D72r8vKCPV+S3Py64uofKHoOCPpeEJuatrXSBCWoz0FtzkBKtrf+haQdnjwyMtJRXLY9Tx63aOdpTNjtWouHvToRvVarMTs7u2L6HhgYoK+vb1PP+wcT9qek2nh1LpfbstvdVixRwdCWjh07Ri6X63C7t3qstUoIM9X6vv9Iy8V2qVhz46uauAkH/pzAyax9vLCquf1NhV+A0cOCYrHIqVOneO2115C1AneuJfQ+J8kOCbQyftfChoGDRs0oHY1WEIfg5aCwQ1K+o1ARaA25UUHphibVI0n3SlSsqd21GDlQYOStVzvJ2NPT01y+fBmKeWo1D5lvACunsDZl8+DBg/T29q75+6yuVK9g/5+VTB/XLF7SLFw0/PN0v8DLg50SSLv9mhkjrLgOYQWCsqZZgnZQlOVC7wGB3X//49TrdT7++GNefPHFToNqT9dt2mAbOmm7DW6VNrhePevhBZlMpmOT0J6+Z2dnuXDhAp7ndZgnD2vG3+tOffApb9htP+Bjx46xf/9+RkdHt3ysrUzYtVqNo0ePsnfv3hXpzI8SYLC8kiShu7ubkydP0t3d3Zk2t4qzdxgh05rdPypJ96/drFWiuf0N03B3fl4yPTvJ1atXOXz4MLb2uf6nCr9bMHDQ/PziVU1zSTPyhqQyBUFFkxkQNIoK29Nk+i20EgQVhYoFTlpCIogDxeAh82Eu39EkEXTvMY1ltVnVjW81mCje4NLtMwRX7plVAZw9e3ZLlri2Lxh7VzB4SFO8oalOGlw/CQHuV74KAU7GTOi9Q5DqEaT6wMuvDdNVKhWOHj3Kq6++Snd395rPYbVoZzVtcDtEO0+7YW/mbnf19N02KTt37hyNRoOenh4GBwfXnL5/0LCf4ZJSMjU1xeXLl3nttdcoFAqPdLzNNuz5+XnOnDmz5ofxUfHw5cvF3bt3s2fPns60eenSJTzP63h/bEYosIIRsvcBe4Cjmvq8ZsfnBLdnrzI/P9/xPbn9DYVWMHpEIixBUNYsXFbkRgVel2D2uwmpbkFY1Wg0TkaQH5PMX0rQiXnM/Jhg4ZIm3SdJdZtl5dINjV8Q+OuYFqqqS99IjiNHXuyYVbUDH/r7+1laWsLzvC05tTkZwcDLgoGXW699CHETdGsNIWwzRdue4YJvpIrFIidPnuSNN97Y8C5lPdrgctHORry+V9enOc9xuUnZWtN3G/vOZrNUq1X6+jaW5/mg+spXvsLf/tt/m76+Pr797W8DIITIAf8L0AP8M631v3rkB9pCfSobttaaIAi4cePGlvDqtWozTfbmzZtMTEys67T3KBP2esvF9rT54osvUqvVOgrDOI47J21XV9e6kEnp1sMZIWB4zgsXNb3Pw82l0wAdatzCRUVtRjP8lsTL3YNCLAcGXjLqRGFpsAwV0MlAYcyiuaSJGgoVS9yMIAkNxNC9x3yQm4tmKh86tLZZlYo1QQW8sXuJ4/V6HYAvfelLNBoNpqen+eijjxBCMDAwwNDQELnc5rxGwEzKtmea81ZrdnaW8+fPrxtxp7WmOg1JQ9O1Z/1Gup5oZyNe38vraU/Y24Wfrzd9nzlzhp//+Z+nu7ubz3zmM9Tr9Uei9x0+fJhTp07xxS9+cfk//3Xgt1r//akQ4re01tvj8byJ+tQ17CiKOHHiBEIIXn311W3zvt0I7qyU4uzZs0RR9MAIs61i2BuVmWcyRmG4d+9eoihidnaW69evUy6XV0An7efXKBpGSKr3wYyQZklz5zsKr0dxKz5Kf08f+/btQwhBs6iZPaPJjQm6di+DQkqakbck9XnzONlhQWPRhBN4OUmqVzBzJgYtQGsKOyRzFwx7pC1+WbqlsGxWpKwvr7AKaI2bMa9R2+3w8OHDHRe+QqHAc889RxAEzMzMcOnSJarVKj09PQwNDW16gbXVmpw08NFaylatNZW7MH9e0SxqvC5BYfdKbnY7rGF1PYw22J6+Xde9b3H5vdKwV9fy6fuDDz7gr/21v8bFixf57Gc/S19fH7/xG7+xJd+edeCrw8Df1FonQohTwPPA6Uf7DTZfn6qG3car9+7du204cbseBomEYcjRo0fp7+/n4MGDD5zctgKJbFW56DgOo6OjjI6OopRicXGRmZkZLl68SCqVoq9riMqHQ1i2w54fW58RomIT85WoiDn3KAd27e7sBFRsVI6WB8Nvmil4ORSS6hFMfMckxIQ1A5k4GU3XTpvarCYJNSoSuDlJHELc1Ay8ZHWOXZ3S5McE0l6HZWGGaWwXjh8/ju/7vPHGG2u+Rp7nsWPHDnbs2GHMqlr0sQsXLpBKpTrKvMfhOXHz5k3u3r27QlkJoBMjtZ+/qAnLGjcrGH5L0rXz3sVTJQYWKl5TjP+QZXxXmhrbX/s1Wc/ru+022J6+Xdd9bErHjdSTcutLp9P4vs/f+lt/i8OHD3Pr1q1tgUeWVRdQbv1/qfX3J16fmobd/tC18er5+fnHmpy+vNoshI36d2/2YrLcw/pRTm4pJX19fZ0TtVyqcvp/q7Bwa5LcWzO4d830vdzXul13P9IsTTcp9Z7mjbdeXOFoN3tGE5Q1Oz5rYXstKOSkmaIHXpYsXFaoBJws1GcFlqvI9Fk4KcHClRghjLd1YYfBrr2cINXb8tWeMvFi+bH1G0oSaBKluHbjMu88t4O9e/du+PXo7++nv9/QN9q2oe3A2jaU1N3d/UjsG601V65coVgscvjw4c57mESapeuahcuauG4m6tEjkvyY6EzRWmnKdzQLlxRRw4h6orpm8YqiPq/o2S9BmKAHLyew7Ptfp/ZdRvtus624bE/flUoFKWUnhPlJNu8nebFYzsN+WC4smGjCn/3Zn13xb0NDQ/zWb/3WWt9eAvJAs/Xn0qM9263VM9+w2x+G+fn5FbeZ25U60671puJ23uRmWAgbhUTaqrdH9bBer0pn0qTjFC/8V5LcrtH7fK3bUEH5pmTiZIVK+jo/9GOvrNi0V2c0i5dN48gOtyCMG4YVMvy6JKxqKpOK3KikWVQIS2OnBPlxi+qMIok0Oha4GQFaEFYVAy/fu4OoTJqA3/WWjQDNRsidiUkGnxvYcLNeq5bbhkZRxNzcHLdu3eLUqVMUCoUOfWwznjPLZfBvvfWWaYw1zeJV06yTUJMeEAy/aZEduscm0VpTm9HMX9AEFeOP0v+yIKyYi2ESaJCaqRMRcVMjfUW6D9yMTdeYTbrvHptmdVmW1bmDmJ2dZWZmhoMHD67w+n5Sop3tUA9vtDYrnBkaGuL999/f6Ld/AHxRCPHvgFeBi5t9fttRz3TDXp0mvvzk2u4g3tUXgOUXis0uNjcyYa+WmW+3Em3+vInpGjwk6D0gAJexsTHGxsY6UMH09DRnPrlM/cQQIhvwub+8b0WzTgITpOvmBQOvmOcX1jTzF41AJjsMdz4wcV86UahIY6U0uWEbKaE6kyAtSRS0edgayxVkh8yxkkhTm9N07Vo/GadSqXD27EX6+5/H7qlt2+vjOA4jIyOMjIwYlsrSEjMzM1y7dq1jajQ0NPTABqCU4uTJk7iuy2uvvUZjERYvKyp3NFobNkzvc7JzN9Gu+rwRGTWKGjcjGH5doBIjKoobGi00WiuCsrmg2xmwHCjfARJF9U6MV5B07bTIDUESsqbDYhsaO3LkSGd5uZZoZznne7sb+JOcsLdLOHP06FH+3t/7e5w9exYhxNeAnwJ+HcMS+QXg//s0Fo7wDDfsNl69Z88exsfH7/v642jYQRAAdLwyHMe570Kx0WM9qGFvl4f1elWd1Nz5jonvGnln7QVWf38/PV29zH/rLo4fs+uLcPrMKbTWnWZVPp8hacL4FyXSNvS72dMKIWDgFUnptpGfd++RVCYThANOWpIdlJ3pGm1MnJyMyXrs2i06Lnn1OcP1bk/uq2txcZFTp07x0qE3KZ5M0Yy3r2EvLyEE3d3ddHd38/zzz9NoNJidne1wf9u+F319fZ1zIUkSjh49Snehh35vLze/pmgsaqQj6N4v6NnfuqtYVo2iNkybOY3tG78V2zfL26CqQYNGEVY1SaywPI2bapleVQTSlti+QMWSJDThxwsXFbYv2PFZ87wa80b8s3z5uXzYWE+086he3+vVk0yc2S5p+ptvvsnXvva19l9/pPVnE9O4n2o9kw277THw2muv0dXVteb32La9rZBIG8ZoNBocPXqU8fFxdu3ataVjCSHWjRzbSEDuo1RYMakxbg52/YhYlzccxzHf+NdXEY1u3vsvRsmPSmB/h2Vx9js3KZ5O0/sCVOI8nuqjetfIsAcOSoSA4nVFuk/QXDKN2XIVhVFzSlVnEixXElaga5+kOmlej+VYdX1OY9mQWmMp3+bYHz58GBn5FE8mJJUn44eRSqXYuXMnO3fu7HB/p6enOXfuHJlMhr6+PiauztCtd6GnBpgKzF3I0OuSwhrBCkHZ3JVUpzWWC/0vSvweKF4zTV4ngNSEjTYUovC6IIkgKIrORU8IiZMy+L8KBeU7BkoZetX4iU8eVdTnwH95hsnF6/ctP1fXw0Q7lmWt4H1vpZ5kw47jeEue85+meqYadhuGmJub4913333gi/84JuxarcaHH37IK6+8smGJ82Zqu5aL6x4/0lz/I7PE2//l9b0tms0m3/79s3iVvez/4UKrWZvyPI+R/nEawSiDr2myrywyPT3N2VMXkDd30jWQIT2SZ+GSDRr8LkH5Tmu6TgnsjBHdqAikBZYN6QG4811NqnelZWp93kjEV19UlrMtXNdF+xrLEzSWtv0le2gt5/4mkeLO+RJn//NtqPbTsBv07Jxi/PUcg3tz902kbSZNZVJjOdD3nCQ7bLjui9cMm0ZYmiRRhGUDhThZk/oTVkBFEtszVgfSNq+1lxVUp8zxB16SFHZC8bqxtEUAo9NMFW8ZVaptU5/TTB3T7Py8WJdxAo9PtPMkPcB/EML7BKuNV/u+vyHD/+UQxnbUwsICc3NzfPazn30snrrbEZD7oNJac+sbmsYC7PkxsW4EVrlc5pNvnSK/9Cq9BzIMHhL3HWfquELFMPaOhVcYYHBogKkTCfOlJtaOKT58/zrJ7X5699qo2z3YlouwFFETFq5qhJZYHgQlKIxLwjJEDW0YD+3XIzBwSmGnXPHYFy9epFqtrmBbCCHIjwluXueBVLfHUVpr6vNGsr5wPWDi5iIjO3cy9sN5MqMxi5VZJmeucvEbhgM/ODhI3uujdENSvasRFvTul+TGjPz+zkcaFWukrVGJolk0MIjlG0fDqA5hWSAtiZsWCAy0ku4XBCWDY2cGBIOvSFQMt76haMxrsiOCZs8tliqzHH7rHQSS6ROKuTMaOw1RzcAvG621RDtt3Hszop0nVd8PzRqekYbd9uRYD69eq5ar3R6l2kKMpaUlBgYGtr1ZP+7lYrtmTkLxqmbkHUlh5zpWqbOznDt7noH628icx47P3D/dlm9pKnc0A4ckXiv8tr5gQgWGXkzR98Je8h/torknJvIXmbk5j5Ih2XyafE8G3Ugh0di+RKDJDUuK1xXSEmSWeZc0S+bPdsCuUopTp05h2zZvvvnmfa9TzwEB1wQzpzWjbz/ehq21prkI5Qnze0d1TZQETDeu8uqf38nw/kLr+bmkC/cWuTO3l5j4qMLSxC0sR9J/wGPHwS5UOcXdjxVJaCZtIRXNkpmsha3xsqAiaBYFKIGXFiAklitIdYPtSip3DJwy/IYkOwSLlzVz5zTSgpF3JDPBVcrlEm+//TZhSTDxbUVjEbr3CkbeXjujU2vNwkXjj1LYsfHpey2vb8uy1hTtPMn6Xk9Mh2egYbeXOw8yyFmrtoPW17a97Orq4uWXX+bq1auPdLzV9biXi+0q3dZMfazp3icYfHXt77l58yZ37txht3uEct1m/IflfU59cUMzfcJAF4ZZYnjCs6cNE6T3gKAyadSNfc87lCb6yewCLRKaYZ1KpUKlPI3reGTcPD1DWSzXpjqjSfevFMaEFTMRuXlzd3X06FF6e3s7ysrV5eUFPbskpRsKLw99z2+zBWmiqc+ZhW3lrmnSQgoyQ+DtrHJ19iTvvrM2tbNR1Cxe0VSn8/S4efb8iMQdaHL38iKn/mCaqKnJFlJkMlmspkccaoRUuDlAtC5esYE/LFuCMEHE6V5BbUZTb2gKOwT9L0qiGtz8ulFK5scFg68Krt66RL1e5/XXXmf+PMycUFgu7PyCXLcRx03N7W8a6mHPfvHAhr26HibaWU4bfFLV3gt9r9dTa9jLA2ofhlevVZsN4l1dbRbKvn37GB0dpVarbSsmDmx4uRg1NMHS+myJB1Vz6Z7sfMfn7p8w2ncQ9Xqdl8YOM/EN4/28OrVFa83UMSOAGXlbdibvpRuGJzz6lvkwLF5R+AWjwtMJSEchpSCfzZCJ0vTmBcppMn+zwdWp01ybA29+Fzt3pIF7G/yoYTDuWId88sHH7Nq166F3V917Bb6SzJ5SBEswcOj+GLGNllbGIrU+Z/jQ9VmjuhSWIDsE/S9LciOChdIc58+f5/C7b6+4+9Ja05g3r0dt3kzOvc9JunZCfQGK51ycxiDjY6BIqC4G1OYjhAxxMgInZRE1HVTQSmnPGPjD9gSZQUESCpZuGlXk+HsSv8tQNRcumkl77F2L3JhxKlRK8dK+17jxVfP75HcIxg4L7HVem8qk5tafmBSh0cOSgUNbegmB+0U77eYdRRHlcpkkSajX64/N67tdtVrte96pD55Sw47juMNf3WpA7aMsHefm5jh79uwKFsp2LjHb8MfZs2cZHh5+oCVqEmmu/h40FuGVn9PrLgrX/NnQLBmFgN1fvp+dkCQJx48fJ5PJcOiFN7jy+5pUD2uaP5UnzGQ5cOhebFbcNOyGzIAgMyQoXtfEgab3OYvFqwm2D0qDkBoQWJYkSSDlZRgYSjN2eJDpC00mi1VuzJznwkSd/v5+hoaGiJoFEhHy4YcrvaIfVFJIRt8ReHnJ/HmjEMyPCbIjZnnppO8XkmitSQIIaxCWTehAc8lg/SpuTflZQWGXIDskyQzSuRNYyxdEK2NP27aUtT3D+sjvMIk4k0cVYc0sDi1XE9YVUU0jccj32dgpCGqKYDGm2aiRyBDf88m4afIDLn7e8NVVoul7TtK9T9BchOtfNYvJwm7J0CGBdOHUqVNYls2Ie5Cr/0kjJIz/kKRrz9qCGpVopo9qZk6CV4Dn/rxY12L3vp+NNbOnYOBVHhhe3J6ugyDoqIOVUium7+2kDbbr+yExHZ5Cw9Za88knnzAyMrIh+eh6tRVIRGvNjRs3mJycXCEmaB9vuzyskyTh8OHDlEqlTmRZKpXqWKIu//Df+COoTsPen2BTzVprza0/NQ1o30+aW+jlFQQBn3zyCWNjY+zcsZPrXzWc5x2flfd94OKmZvq4MWRqQyEAc+dNPuPAQUkSwtJNRXZQEpSNMAShkAIs19AYVSTw84L6vCbTb6b0pOowuLObsXf6O7aoExMTzJy6Y0IUvjS6YShMSGNv2v+SoLDDSL7LtzWlW7r1dYHlmcldY+xRk9C8zp1jWAKvAIVdBnJI968d4rDaF0TFmtKEZum6IqyBm4HBV8wysbEAk58Y/rS0wfI1ccNwq7VWCEfjZczzaS4KdGKTzjlkuzOoWBHoKiVxm8nzCp8uesdS7PlMHjctmDmlKV41atAdn7XIDptQ5uPHT+CSJT2zj6lpTW7EeHuvF0gRlMydWG1W0/u8YOy9+y/w61VQ0lz7A6jNGnl990PEps1mk48++ogXX3yxYwuglOrwvrfT67td3w9e2PAUGrYQgrfeeuuR36DNTsRKKU6fPo3WmiNHjqwZW/QoE/by5WKbCbLcgL9arTI1NcUnn3wCwMDAIOrqKJWrPuOfM0KLzdT0cQNXjL0r73O5a4cQtyfXmdPGFnXsvXuLxOU1c9LQ8IaXQSGNBTPB9u43lqhzFxJ0ArkxwfyFBCcFKgHZgikFEhRIR6ASg1mrRBOUoau1BLVtm6GhIYQQzMkZxsfGSJJZPvjgA2zb7gh21vvgLX/mbk4w/IZg6DVNs2im5rAGSRN067rb9rF2fNPI3By42Qd7Wq/2BVGhZO6KonRLkUSQ6haMvCDIDLYa9ceKoGIEM04aooYiWNJopdBS4+XMxSMsg4qM+MVLS0BgOZAZtxF0U57oYnBMY4+WqYhJvvunl9F3B0g7eUZfyTD+hm9eW6X45JOjuJUBxOw4DW28yXv2rz1Va61ZvAx3vq1Bwu4flQ/0Ql9dxWuaG181/7/vzzzYRx3WbtZwb3G5XLSzmjb4KKKdHzTsx/mg2+AtsBkMOwgCjh49ytDQEHv27FnzxBZCPLKHNay/XMxms+zfv5/9+4045cLXF7nxJws4OxdJuzbOwhA9PT0bWkyWbplb2579gv6DK7/WDlZoG+fX5zSzJzVdu9aejCpTmtItRf9LssPY0Foze07h+NCzXxA1TPPOjQoaC61pVZilnHRaf9cgbTr8a78gCCpmuvXy9z58t27dYmJighdefouwaLHn+Z6OunB6eprTp08TBEHH03q5MZNgjfdNClK93Cf/3krd8wWJeHnPm8ycgOpUAtrsF7r3GMFLbUZz9yPTqC1X4GYFUcMEECeJAmn41MKGqAZJUyCFxM8KEAIpzZSf7rUoTyiihqZrp6DvBQutepg52Y0VKBgNEeOz3Gle4uZ3Y/r7+5m7u4Q/txsrGiA1aCbl1XdX7YqbJmGoeM1M4Du/INaUsK9VKtHc/Q5MHzfpQXt/kjUv9sur2Wzy4Ycf8vLLLz/UKW89r+/Vop028+Rh9f2Q5wjPAEtkq7VRSKRUKnHixImH4qRbZXC0lYttYcFGjlO77RJdGeLg52HXj40wPz/H7du3OX369Irg2bUubM0lzc0/aS0ZP7tyyTgxMcGNGzc6cE8SmlxGJwOjR+5fSCaRZvqowsubRWS7yhP3zJ2kLZi/ZC6M+VHBzJkEJ2N+tp15iKYzfTbmNaluQxcMq+YC6OYMZHL58mWWlpY4cuQIS1cFtSkjX7ccQSqV6ngbx3F8nzHT0NAQ+jGq2JRSnDh2ClHO0SOf584ds0js3iPp2m04zNUZzcR3DfRhuSa0t92o40ihdEtOnhHEDYgqhqbnpg2vGi1w0oLciFGHLlw2r/2ONyR+t7kQt+92+l+S9L6QQlq7gF3U6w2+/XtnCK8PshRX6Xtlid6DBaTfD9yvZly+WBx5WzL46sbTcoKy5vpXjJPiwCuC8c+yrvVtuxqNBh999NGazVprcxe0Xjr9eqKd5ZL5h4l2vh8S0+FT3LCllA8ly09OTnL58mXefPPNx/JmLlcuSrm+gdHyqtzV3PhDyA7B7i+DZVsdbLsdPLse7r16ydj2tm4nxpdKpU6UF8DkxwYm2PtluSYPd+6sJmrArh82cV9glkvzFxWpbkFu1Fh9Vu4aCll9oY1dtzBjxywbhRSgwE1LKoGiMG4+UFHdmPNbnuL06bMAnaBiv9sco7GoyQ6ufG62bTM8PMzw8PCKMN6LdyeZymYYGhpicHBwWzjzWmtqCwnH/+QKbnOE7q5eREEwdMj8/gjTuIo3FFHdiHb8LpMGX5vTJJHBZoVtFpA6gsaigMQwPmxPopUJ9M0NC4QlKF4zv3v/i5LuPYKoDre/qahNa1J9guE3793tANRLEd/6X26Sifez97UCo+9CQ5WYnp7m2rWrWJZ17zVJZTqLRTcPB35akBnY+DCydMPsVVQCe35c0Pvcw3+23azXCkGOA83V34fKHTj01wzr5WHVnr7btMG2aKfRaKzw+l4u2vkBJPIprvY0VywWee+99zZll7nR2opysbFoTl43B/v+LPctfVYHz1arVaanp/nkk0/QGqwbe5Clbl78837nNrjtGOc4TqcZgpE/F6+ZCSkzeP+HpLFouMPdewXpvntfX7xmEtVHWkEFxesJQhphxfTJBC9npmlhg7RkZ6EnpElDh3u3zkkAwlYcPXaCrq4uDhw40Hl+qd62A50mO7j+a7b8NSmMjtHt2ExPT3Py5MmOp/XQ0NAD49HWqqhhBELFWzE3Lt6l0N3P+CvdFHYaCp1OzHNbuqlM3FnaBDWENUVttjVRK8OntjPm+6MqCCWxbIlTEKAlQkK6R5Dul5RuGnVnbljQ/5I0xk8tAQzA0OuGFbLcgnXuUsjH/2GCnu5h9v9wnr4XzAXSx5hVvfDCCzQaDWZmZjj10UUWPs7gRt2Mvupz4Ms5HG+DEnKlufsBTH1iLhr7fpJ11bLL60HNurGoufTvoVmEnV80Ap3N1kZEO7Zts7S09IOG/Wms5Zas77zzzraLVdZaLm6korrmyu8CAvb/NBviD2ezWfbt28e+ffuY+Djg6lQNxq9x/Oo8/aV++vr6uHLlCiMjI+zZs+feY9U0dz8wxkxtW9QVv4PSTB1V2D6d5HMwmGfxqiI3bJpT1NBUJs103SgajxLpmKnJ8jS2K4maGh0balxcN/i10xp8g2bErTu32Pv5YXbs2LHiOUhLUNgpKV5VNPcaE6OHlRQmWWTPnj3s2bOn42l948YNSqVSRxq+HpyUBJrKtLljaMxr4jhhsnidXYd72H2oF8sx/PLFK6qV4m7CFtycIKgqqjNmok6UBkyjRplGjZZYUuLm23cchgdd2GFoeXPnFG4Gxg5bJlF+UTPxbSOAyY0Khl5bKWSKGpqb3wy58OFdxp7r48UfL6yYupeX7/tkGzvJ3NhBZlCRfXmJun+bb393gUzm3h3JelqHsGogkMpdTd9Lgh2fv3+YWKse1KyLVzVXfs+cDy/8LOuqbzdba4l2giDgX//rf82RI0e25TGe5fqeatj1ep2jR4+ya9eu+xrERktrvW6TXy8g92GVRKZZRzV47i+woea0vMp3NPMnHHa93sWuH+lGKcXExATHjh3Dtu0OfbCd4zjxbUPhG/+MWJMzW7yqaRYNw2Q5VLJw2RgS9b1gLkJLNxUI6NolmDmj8LKmqUvHTNfSNpN13IBMv6SxYCZRIQS1Wo3Ll68zNDDGjh1r0/Z69gnKd2DqWML4e9ZDPULkqqXjak/rxcXFTrK87/sMDQ3Rmx8kKXlUp3QH0nEzgszOkMvTxzn0mefo7+9vOeolVKeNzanfbXDboKypF1UH+tBa46QN8yOug47NQtHJGpWiis1SMb/T3B0sXDTHa3OqUTB9UrF42UAobQHM8ql66Qbc+nbA7Vt3eOGLvex7t7C+62KjpVi8YfI2d37ews32AX0mQ7JSYWZmpnWXpjspO4WCkdeXbmmu/yGoGPZ8WdD7wsbOzXazfuWVV1akE2mtuftduPMtSA/CgT+/+fN9o9Werv/G3/gbvPvuu/zqr/7qY3mcZ6k+9Q273WAXFhY4ffo0hw4dWnECbabawQNrbaWXLxc306y1Mh+I2izs+6nNqxmDiubm1zR+N+xoBeiWSqVOYnyhUFiBezPXh5wcYf8X03j5+91+ooZm9qwJy82NLZOK1wyfubDTsB7iwEyiuWGDscZNjTdggnqlC35BElYV0paAwSYrdzVuXrC0tMSJEyfYvft1rGj921TLFYy8aXHng4Tb304Yft1adzEFsAZJ5N6XhKC3t5fuQg+7+mHxToOZT8pcnZlGaUW+N8XQviyD+7OEssrx48c49NaruGGBux/HNIoaaQvSfYCEoKgJa4okbjE/UNgpc4GKGqDCe3anTgunBsgOSdJ9gqXrRiGaHRT0v2yokZVJzfQxg4V375UMvLLS4yNqaO5+oJm/FjBTvclbf3mI4d3rc9TLE5pb72uSpqH2DbyycnkuhCCfz5PP59m/fz9hGDI7O8u1a9coLZWxJsdQtwbpHU/z/E9ZG2ba1Ot1Pv744/uadRwYvvbiJeh/CXb/+MYm9a2WUopf+IVfIJPJ8Ku/+qs/kKY/69XmTt+9e5dbt4yl5KOEq7bFM6sb9lYDcgFufxOWrml2fO7hHNbVpWLNjZbgZfeXjNDh7t27XL16lXfeeaezdGtjvM2i5tx/bBL1F7laPM+1b9PhNrc9MGZPmeMNvbZySbpwSSEk9B4wJ315QqESTdcui6VbCZZrqF7CEkjbMCEaSxq71XCctCAJNdV6iVsnz/D222/TnPRZuqHXTQIHwxwYe9di6qhp2tlBAyOs9h5Z8/VJNFHNsBqaS9AsGnaL1iCky+BIP3teE3g9MYu1WWZnr3Dl6CJRQ7Gz52VKpzIkYYKdMontOtY0ipqoYeCQRC1r1LaZqFUoAInjmWaNFmht8hYLOwW1GZg5pXDSMPq2JDskiRqaO99NKE9ovLxg1xetFXsDrTVL1w2nu1EPmPfO897/bj9d3YV1z4vJj0yKvd8t2PvjK/cQ65XrmtShga5Rrn1FMzvRQA7Nszh2kpOXnc658qBl7nrNulk0eHVjweDVw29tnXm1kVJK8Xf/7t9Fa80//sf/+PuiWcOnvGFLKTl79ixxHK9gR2y12heA5UvKJEmI4xjbtjdtIzl9wnCgB18TDL62+ZP3znc09TnNni9LvAIrIstWL1JVorn9bUUq43Lozw5jp0Y6YQTnz5+n0WhQsIdJro6w4430Cj5uUDELuO59rUSTxKj6Mv0SyzXLo8ygoaJJz+DPy5upsIyv81KxzFJxgiNfMlJunVOtqCvwu9b/PVPd5la+eF1TummwYgE4WSNGsVwz2S42NbFMSEIDBcQNA02AwUq9gqB7r5lwUz3LqWgu6a5RqHssXXQY9kYp3QiY5jJ+XpKXvURTeXQkUYkiSYxC0U4ZbnlUB1UTxjbWNtOykAIVGXVq125BEsDs6ZXwhxDGa2T2jLlIDhyU9D4nOowcMLuNux8aV0CRabA0dIL3PvvquvmhjQWjWGwsavoPCkbfvscW2kiVb2uufcWIeF766TR9L+4EdlKv15menubUqVMEQdCxEOju7u40w3azPnTo0Ap1avGasVcQsoVX73p8jRpMs/6lX/olKpUK/+yf/bPvm2YNn+KGHYYhlUqFQqHAoUOHtuVqvjw8d7Ut6mZPiuI1zcQ3oXuf4bFutuYvmIDWodcE+Z2aU6dOA/DOO++s+VxmTxtL0J1fkB3TH8/z2LFjBzt27CCOYs78xwrVoMjZmWN0x4UO33vhskBY0NO6A6jNmPDYwg5JbdbI06UDutUevZxFEt4TzFguXL16jXLF5qXDL+J5Rv6Y6mnh2bMKv+vBFzvLEfQ9J+jdL6gvmMYUlE1TDqsaFAQKIg8sr+0dInCzxslvPQVjVDchwRMXlliYLrNj1wG6Rh2kC83FASoLAdXFOuVgEVs6eJ6Lm5U4riRpQLMqkJgQAS8jkI5EhQCCwk7wctIkntfpsD+ctFnSTh81iTKZIcHw63LFRVJrTfEaTH1iDLcyByrcbp7kyNtvrcl20Fozd9pQNS0P9v2EJL8Jhz2tNJMfw+RHJuFnz19gxVS+fJnb5sFPTExw+vRp8vk8XV1d3Lp1i9dee63TrLXWTH4AE980IRUHfubx4dWd30NrfvmXf5mpqSl+4zd+45nw4n6S9als2G3pdTabZceOHdt269WesLe6XGxXdcps3TODhmu92Z+vz5lMxtyYoO/VhI8+OkpfX9+61qP1OXN73L13fZvM8g2Jp3O8/pNd5EbH73GbT11D3xhj8AWPSHdj4VOaULgZE0U1fVLhZgVJoLFsgZAaL2vYFO0otMmpu4j+KuM7XkLoex8gyzUc4PIdTdcuvSYXfHUJKcj0s8I7u13FukN3+uEf0DgwDnyVKUVzSVMqLVFVCxz8wm6ksqjNGRxZJRqUxHfT+G4KO61JSIjqMbVFA+U4jotXcHE9SRKbqTozYOwAlm4YVo2XE4wdkWT6hREjnVAsXjFLxdHDprEuf9/CqpmqK3eNijD1fJHLt87wzjtvrwlHhFXjG1O5azIxxz+7OZfCqKa59odQmdD0viDY+QUe+F6s5sG3J2/P8zh37pzJt+weZPabORYvQd+LsOcnHi9eDaZZ/8qv/AqXL1/mN3/zN7/vmjV8Chv2zMwMFy5c4PXXX+f69evbHhPW5nludrnYrqBk6ExOBvavwbV+WMVNg1vbPgy9F/DBhx93LGDXKhVrJr5jMNORdYz948BwfTODgtzoSm5zdzNhsRHgDExx9Og1VMPCn9/D2KEMcTNNUDGTdn1eddz53KwkrCUgFTdu3CaV8jn46qtMHUuImyvFTN17JbXZhNlziqFDcsNqu7XqQT8bB8bYqDZj4AKtzSK05k8SijIHBp+jdhfiwDA9lDL8aq0VVsqoNlUo0E0HWxtKXiJC6mGZxTt1bMujazDF+ME8cdli8mOFtE1MV9duI7ApT5hmHTeNFezAwZVLRa01i1dg6qgCbd6vuHuOS5cucvjw4RVmZO0qXtXc/pa5w9jxOUnv85sbAMoTZnhIQtj1o8Y4azNVr9e5dOkShw8fpquri2azyZ0r83zr/7lIbW6esc8ocu/kQPYBj6+Baq35R//oH3H8+HF+67d+a1vsLT6N9an5rZf7Z7ftLh/VE3t1SSlpNpvkcrktBeTGTc3l3wE07P9zZhG3mdL6npx44PMVjp48/lDWy/QJ49i3+0fXVjMCzJ3TqBgGVy0a2yrG/v0+Ay/u5QB7mTrbZKpUZWLpApe/BplgFGfIR6kMSIFtmeVnFMXcunWLfCFPb3cfQhh2SaOoVlAjvZyg9znJ/MWEmTOmwT1smbheLQeCtNIEFajPa+rziqBEi3JnMGWvILhy7ibhos1g9wHqsy1aZqKIAwVo7IxGtmiJYU0gaBkU+SaWS6s06TDN0CC4w03mphc4/p/LEEt69jjsfrNAV1+OqArTJxTVKcMnH3tPkl7FuAgqmrvfNdas2SHB6LuChcoUVy/fn2wO5gJ059tG3JQZFOz6YfFQL4/ltRoCOfAzbGgxubxqtRoff/zxChvi5qRH8WujDPfCvv+TJs4tMDMzw8WLF0ilUgwODjI4OPhIy//7fhet+af/9J/yrW99i3//7//9YxHCfVrqU9GwkyRpef9aK/yztyN1pl1KKXp6erh48SITExMdSfhGUzNUorn6n0yO4YGfWd834UE1c9ykx6RfLHL57lnefvvtB6q3ajMG5+45YPwp1qqgpFm6punaI+4TXhSvGa+PNntFK00wbzOyr5uhV/uYOh2yNFdlZmqO2swc6W6HnoEcfsXn0qVrDPWO091fIFgyU3WqR1CdNX4nqWUqua6dEq3MAm6ilNC9R5IdejgLZHmpWBMuaYoNA3M0i4bJIYTAy5tJPtVrLprVacXl41NYpOnr60HFmihKSGIjeHGyLf54HaLA8KgtS2K7xtVPIIibZmnZe0DipGH+Qhq/nOK5g4LC/pil5gyXr12k/FUbZ2mQXD7Hzrcy9B5YeRehlQkdmD5hQnJHDwt6Dhi2z82bN9dMNl/uAzL8pmDo9fuj3B5Um4VA1qpqtconn3zC66+/TqFQWAevlkB/x5WvWq0yMzPD8ePHSZKks7jcrAp1eWmt+bVf+zW++tWv8tu//dtPNMXmWaxnvmE3m02OHj3K6Ogou3fvXvG17QgdaN8eJ0nCyMgIY2NjVCoVpqam+Oijj5BSMjw8/EC6k9aam1+Dyh3Nni+bwNjNVvmOZuqoJsrPMi+v8+7hdx94cqpYc+e7Rj03vEYgQbtmTiuExX23wkmkKd02Krs2HtoomoCC7LCRnEcVweDOAkrniboUzaDOUmWas+9fodcfJwoTlAxJIpsk1KT7BJYLpVsKv2slbtu9W+J3CRYuJcyeS5i/aIKCvRzYKdHKOjTydp2YW/i4abxOwqpZPjYihW8nRvgyKEj1SFLdhsVRm9XMnVPEoWJqcppU2iefyxPWE5TWCEvh5s3xkxY9T0iJbZuJ380akVFUM9BL106Dvy9eNVOxk4aRNyTZEYEQHnJqnHBhlIylEHurRL23OTM1R7aS7SgLVd3hzncN0yc3Khg9bB7n1q1b3L17t5Nsvvw9nTpqggLcPBz4c2vbCjzwPOqwQLYGgcD9zToJDb964WILr/7xtS8A2WyWbDbL3r17V6hQl5aWVpiabWZC/hf/4l/wO7/zO/zu7/7umpDR91uJhxgoPdUo4rYA4+WXX17hrduua9eu4TjOI6kaHxaQ27b9nJqaIo5jBgcHGR4eJpfLdb7/7oeayQ81o4cFI4c3/wEJq5qLv62YXZyi8O4sb7z16kNZKVNHFXPnNHu+JNcV49RmNbf+NGHgFdlRL7areE0xe06x87NWZ7M/dz6hMqXZ9XmLqAZTJ2P6X7CoTBkednmpyt3KZV4/8hLNJZi5VqUYTSCXuunaZTH+XB9U0yxcUXTtknTtWFuA1FyC6rSRZYe1lQEDy0vaxiXPzRh5uPJDuvo8pG2m+Pq8prFgHP+0Ai0V05MzpP0cruWh0FiOMlRFbZSmRCbAQYpWo84YWlxUA7QgO2wWt+UJo/QUAnr2G5MmaQuimmbmlKI8oXFzgqHXzd1C+3crl8tMTU4zcbRBeLtArpBhz+fSjL6cQQjBtWvXmJub46233lqxNGssGBfGxoKm70XB6JGNBwxACwL5CCY/brFAfmLzEAiYhf7Ro0c7zbpZ1Fz6bWjMw44vwPDbm1+itw28ZmZmmJ2d7fiiDw4OPtCU7V/9q3/Fv/yX/5Lf+73fe5o+IY93k7rJemYbdlsg8uabb677Zt26dYskSVb4aGy0lntYb9QWNQxDZmZmmJqaol43cVep6ijz383T/5Jg149u/mRWiebS7yZcO32XnT8R8PJb+x96jPqc5upXFD37DTthvd/v5tcUURP2/fhK3FhrzY2vJ9i+YMcPWZ1/u/WNBL9LMPSqRWVSsXA1YfQti9lzMXVVZHG2zAuHx8j3pYibmrkrIflhi6WJiHKlTNm7RaNeJx/vJJX0MnggTfcu68GBAUqThMZQSisQwnCfpWP+bDNR4gYszQXoik1zSaMMDG3ohloRNGOKMyVSqQy2I5GesUdNIojrApTEsow60bIlTsZMiVHNyLIz/cbYvzFvJPoqgvwOQd/zBtPWiUm4mT9vPhK9Lxgnu9XS//q8ufNpFiE9EiF3zDK3NNXxeBZCrJistdbMnoapj01W447Pi037boRVo6at3DHNfsfnNw+BwL1m3fZSb/OrEXDgp7ePX902q5qZmaHRaNDX18fg4CC9vb2dQeXf/tt/y6/92q/xB3/wB+ty0p9QPVMN+5mDRLTWXLx4kXK5vKZAZHlZlkUYhpt+jK0qF13XZXx8nPHxcZIk4dbpBc79To04PYnTGzI7O0xfX9+m6Ea3vh1y8ZNJnv9JnxfefnhkmkrusUIeBIVU7hrBy8hb9y/56nMGRuh7fhnVrGIWXel+84Fp5xJKGxYWitSZY3xsL67nUi/GLetQI6bp3uGir3UzNt5HegBmZ+a4c2aGmW+Bf8pmcF+G0f3duP7976WQZoq2l93tqsS42oUVCMqKoKyIAwjihJRvIV0QaOJAEVQ1YTMmiiNy3Vls34h4kiYEJWFMmWyJdARSCJyssTyNamZZm+oWdO+TJIG5awlrkOkT9C0LdKhOa6aPK8KKgTYGXzPimRXvS2y8rOfOaxy/nVjuAzvYrcc5f/48xWKRbDbLt771LfL5PD2ZIYILA9Rn5JboegClm5rrf2QgkN1fEvS9uLX+srxZ53I57nxHGz+Qx8CvTqVS7Nq1i127dpEkCfPz80xOTvL1r3+df/7P/zkvvvgiH374IX/8x3/8tJv1M1fPVMOOoojjx4+Ty+VWWIWuV1vBsJd7WD8KjzMqS0of9bPreXj+Lw1TqhnjofPnz5PL5Tq3fA+64Nw5U+HEV2Z57jO9vPBDXRt63NnTptHs+uL6rBCtNHNnFG5erDkVlW4rLHelr0lj0UyObXZD3DTUwvPnLtBsZjnw6gEjZGlqGkvmNU91WVRmElJ9Bk8u3UmQlsXwyBDDI0PU5hMmz1eZv1zj1ombOBlNz2CO3sEu/JQLrelZxaBCc8GIG+ax27d2lmv+8zwgFKgwIawYYU+iFBqFTGnSrglyTBqgQolA4DgSIcwS0M2A7UuiqllY+gXB4EEJAubPGUm6lxOMvmOwayEEYU0zc1JRudNKL/+sRW4N+Kk6pbnzgSKsmISe4Tfv0fm01p1k8/fee6+TbHT7RJWrv9ugWr1N7qUy+b15QjWEw8Zu/VfYofYK9v7E1lN3ljfrtJfj8n/kifGrLcvqMEsOHjzI3Nwcv/7rv042m+Unf/In+fEf/3H+/t//+99XasYH1TPTsGu1GkePHmXv3r2MjY1t6Gc2S+vbiof1WhU1WvQ9zK2ik5L0pfro6+u7h2NOTXHt2jVc1+0sLZcvTe5cm+Povyux9+AIL3x5YxSoZlEzd8YwPh602CzfNirBsXfv5z0noVmide1aGcbbXDKUuLZjXtRU3Jm+SW6nZHh4CMuWQEISLXsdAoXjCyqTiu6dFjqRFG8mNMua/JhFps9i32fyBJWcyYica1KcrXDp5hRaazLZDLlsDtdzsVwMSyNrJOZam6YUNTRhXZMEmijUQIzSyuQ1+hohNSoWRFWBjiVSGLaHFOb3s9Pm/YmqBif2coL+F40X9cIlTWXKCFwGD0kK4y3JeayZv6SYv2DCIgYOSnrWgD/iwCwJi1c1bg72fPkeng2mWZ86dQrbtnnllVcQQhh3vW/B0vUMO57LsvOHB1BOk5mZGc6cOUOz2eywK9aLjAvKBgKpTm7ODnXNc6Vc5tixY7zxxhs4cY5z/ys0Fp+MH8jq+qM/+iN+8zd/k69//ev09fWxsLDAd77znR8062X1TGDY7RzCV199dcMJ2gCLi4tMTExw6NChB37fRpaLGy0Vay79B6jPGKvU7Dp0unbVajWmp6eZnp5GKcXQ0BBJpLj6+4KRnl289L938PIPfz5aaa59xTjAPffTcl0rUq001/7QMEP2fOn+FJylW4qZUyuXjQC3vhnjFQRDhyyiKOLD375Jz0iaF94bYfL/3955R8dZnun7+mbUex2Nmi1b7h3bkg2YQEI1trGRbAIEEghJCCSBhDSS/SWbcJIl2c0mWbILm2ThsGE3sLFkXKkOMcEUy3KTsVwlq8+ojdr08r2/P17NoK6RrDKG7zonJ+cYab53RjP3vN/zPs99H/EQn63D2akSEa/g7RVqj1MmsHissvsiPlOHxw5Wk7RpjYxXiEzUERGjoI9UAnasQhU4rC6aTW20Nltw290kxKWQEJ1KhBKN6pVfLKoPVFWg+lS8qhcUlbDexxE+2XqnuhV06NGFK+gVHSgK+jDZnhcWpeDqlr4jkfEykzEyUYYGdNUJFB2kzJG1a12Y3O33NELzcRWPTZCQqyNj+eAkciEEXbXSrMnnhLTFChnL+7cpqqrKsWPHiImJYcGCBdJlsU5Qd0Dgc4FxtULG8sHDQP6x8ObmZjo6OgLxaP7uis5qGYqr+iDvekhdMP73sl+sV69ejbc5LuBfPXfz5PuBDGT//v08/vjj7Nu3j4yMERItph6tht2Xmpoa6uvrAzmEYyGYHXYwAbnB4m/fszYJZq9XRhVrgNjYWPLz88nPz8fpdHLs2DGay8JQO1IRK+uw+1KJEKP3qbafk10RueuGF2uQu2t3jyDn6qFr89ZG6bQX2ccITvXKnWxCrg6Hw0FZWRlpKUvIyEhCp5e39z6n7Nbw2AVhvTcEsWlh2Nq86KPA51LoavAREaMjYYZO1pC7Bd0Nw/199ERgICvagC9MxeFw0tFhxe3uJDIiisjIKPR6HegEuiiICBMInzQtcveA2+4lIiKSiCg9OkXmJerDZZCCPlLB1SmwdUmhTluuJyoFOqvAdER+mSTmyR7rsEj5Grm65JSirVkQmSjNqIZqqXNbBU1l0qwpOgVmXa8bVIpQVZXy8nKSk5OZO3cuPo+g8T25Y5fli+Hd9QaOhXd2dmI2mzl39jyus2kopkzS82JZvCUiqESY4fhoZ72arhNx1L8DscbJ9a8ejgMHDvC9732PvXv3hppYhxzTJtiqqnLy5MmA09546smjDc74PazHY4s6FKZD0H5GkH1lcFl3ffH5fJw8eZIIazozImeRukEQnt9CdXU13d3dpKamkpmZ2e+k3I/HLjAfFcRlKSSN0BAjhBykiUyUI+iD1uCWJv4pcwZOPMr/d6lWjn9whGXLluGsTkT0hshHxClyRD1PwVmvEhmnw+1QiUqEhMwwrC1eFL1KmCIjtFxWuZMOi1CISpG7YdUnBVeoctcsfCA8sgceHURERhAeFi5zOnUqSoQAxQdCkfXtHh2qB3weH06Xg5S0JPR6PcIne78jE+UO19EmR9OjEmWHR1QydNXISUSfRwYJpy746NDQ1zu231ElD1mNV/Q67Q3Y+QpV0H5W/h1ADrT447oG/p0PHz6MwWBg9uzZWE3SB8TdAxkrZH072IEhRVFITk4mRpeEOL6Azh4P+gUWHLlnOVTh7ueoN5b3dldXF0ePHuWKpQU0vRFLx3lIXwKzbpl8P5CBvPvuu3zzm99kz549ZGVlTem1L0emTbBPnTpFTEzMsIZGwTDSoeNEHS76aT8jDXtSFypkFo7td10uF4cPHyYjKRf7h7lEGSD3Kh06vUxLUVWVtrY2TCYTH374IQkJCWRmZmIwGAgLC6PpsOwzzl4zOPm8Lz1NclebvXboQGBbi/SKHrhz9LoEDoeDurMnWb1OtnQ1N3oDviDRyTocFi8KOsKjdDh7ZL3b2uolwRhO8swIHJ0+HJ0+dEK2xAkV3PZeoVYJ/A8hUIXo/TIQ0gFQEeijpOe0UGUpxOdU8Ll1qB6Bx63iclpxe50IBOnpBvS6MMIiZJSZEAq2FnkQGZ2sYFiiIzIJuuulUHtdEGtQSFv4UeeHUAUd1YLWD2VbYfJshfSlSmDH3ReHRQYL2NsE8Vmy1z4yfvDPeb1eysrKyM7OJjd7Bo2HVFqOywzPuZuCuyMbiOW8vKsDWLAlnJS5RsAYGEypqanh+PHjgXg0/3tmOPxivWR2ATU7YnF2QN6NYFw1tfVqgLKyMr72ta+xc+dOcnNzp/TalyvTJtiLFy++5MOE4QR7og4X/VibBDVvSpP7vOvH9sb2OwsunL+IzvfTAMi7of8Blk6nw2AwYDAYArfBJpOJc+fOobcnIc7PYPbVsUQmDJ3J56f9jEp4nEJC7tDrs7fKXt+oAccEzU2ttLT0sLpoJQkJst4RHicPE1WfFEF9pEJ3g0ryLB2WGh9eB+gioNvkISpBT1SSDrfDKyt+iuz8QBXodFL0hQo+nxxE0evkDhyl99+9crLRZwd8OlSfgiJkTToiWiE6DiztdvQinJi4aDqsLbiULuKik4m2pBIVGU1suo6kWXqiEj8yYfI4ZNdL1ur+ZQtrs6D5mIqrSxBjUDCu0A1ZXlC9guYTgrZT0tI09xodSbOG/vt7PB4OHTpEXl4eKVHZnH25dwhmYe8QzBj7olWvtOdtqZBeIvm30s9LZGA8mt998fz580RERASsFfp6evjFek5SIRdLY9CFwaK7GJNN60Rx9OhRvvSlL7Fjxw7y8vKm/PqXK9Mm2BMhpANr2OMNyB2JgPteHORvZEz+F/7D1FWrVtFzKg5bi2DWTboRDxn9t8HJycksmL+Qk39x0BPdTbX1FLXv6jAajWRmZg4ak7e3y2BZ4xXDO+LZ2wUxqf136dXV1TQ2dpCbu4To6I9G4aOTZd6iwyKDDJJm6Gk/78XaLEiaoaer3ofXLgc07B1enN0Kam+5Q6eXvdXyMrLjQ/UJcPaWRLy97XdeRe68vbK9ThHSx0MfIevROp0OnxfMjc2ERejIX2YkIk6HszMJq1nF7rBhC2uhWTWRJGKxVWdDezI+pxyDz1iuIyb9I4F19QhaTqj0NArC45RBeYp96WkUNH6g4rZKT/PMVcqwZwcul4tDhw4xJ38uumYj545Igc9frxtX+KyzQ1D1ivyCzbhCIWcdQ2Zz+unrvggfHXQfO3YMj8dDRkYGsbGxnDt7nlzfWhpeiyYuS9argznwnmgqKiq477772L59O3PmzJny61/OTPuh46Xgn4KDiT1c9ON19Sad42/fC/4x6+vruXjxIldeeSWulkiaT6ikL1ZInh38Y7R+KFDckSzfmEF8lhGn04nJZOLEiRO43e7AmHxCQgKWc9JvOmnW0I/vdUozoeRZH/UHnz59GrvdzvKVK2g50bsr7iUqSSE8SqGrTqavRyfrSMjR093gw+dWSMwJw9Hhw9mtIlQFRZF/B1VV8PaWPISQtqCybg1C1fWWRaSYK6oMB0AvAxT0YbK+q/pk3drnVTG11ZOQG05ObhZWk6CnUUUfrpAyW8+sGUnoI5LoaphF4wkb5hYbDvUCMTNcZOUmo08woihR/erUih4My3SkzBs6oNjjEJgOy1DbyMTBrXoDcTqdHDp0iPzsRfSUpWJrVknOV8hdpwSCJMZC+2lBzVu93Rq3KSSN4f3ip+9Bt8fjoaamhhNHTuE8lkN3Zxcz1riYXxRPRNTUf/wrKyu55557ePHFF5k/f/6UX/9y57IWbD8TfbgIve1x+8DZ2XtyHuSJvBCCs2fP0tXVxVVXXYVw6al9S3YHZF8Z/LpcPTKUIDHvIye+qKgoZs2axaxZs/B4PDQ3N3Pu3Dl6LA505+eQvSwGJWzoHEBnV29aTJISaDmLjIxk1apVuHsAvHgdAnpvuxVFIWmWjtbTPjprVZLz9CRk6dGHQ2edj7azXqKTdMSlhuHzSW9prxN09Iq1IodfhE7GZukURdol66S5EyjoogjspFUveJ3g9cp/i8lQOVtzktSMbKI96bSdkaEK6Qv1xGcpKDpZ+rBckNOJcQmxzFwdR6zRGNhhlh8+gtsUQ2RXJjFR8WQsiMKwRDekkAohsJyTh4qqFzKWy5r2SDtbu93OoUNlnhQYOwAAOVJJREFU5IQtp+1AIrowOdCUMnfs7z+fW1B3ANoq5TTl7Fvol1AzXmw2GzWnWkiv/TQiJpyU67pxpzfy3getATvUgTMCk8XZs2e58847eeGFF1i8ePGkX+/jyGUv2H6x1uv16HRDH7aN5zHrDsgWubwbhq8JD0RVVY4fP054eDiFhYUg4MJbUgBm3TA2O1HTYTm0kbV66N8JDw8nJyeHnJwcmk96qW200R1Wx9/+dpTk5GQyMzNJT08PHLhKUQZdlIdDh45gMBjIz8+XjxUryxjOLkGc8aNrxKQpxGXo6KqThvtJM3XEpst6tdWsYmvz4eiQP6vopK+HLEyDUKX/htx99+Y+oqCPlJ0IQgWfW7YJClWOwMcadMSkgsPmpuLdapIjZxOpSyAqXSExV9ahhQ+66mXSi8ch7wSyCxRijR+VemJjYzFEzobuWdi9XrypXTiTz3LO1o2lKi0wlOIvmTks8kDZ3iq9qrPWDraiHYjVauXQO8dI7VpFT2cMiTNgxrWDe7aDwd4qSyDOTshao5C1ZuSwhmDp6Ojgg93niW9cTVh8BPM+B/E5SUBS4Dk0Nzdz5MgRfD5fYOIwMTFxwg8gq6qq2LZtG88///yocxMawxMSgzPjxefzUV5ejqqqZGVlYTQaiYwc+WAuGJqPCereFhhXKeReE9wb1+12c/jwYYxGY0AIzUcETYdVZl6nG9OAQ0+T4OKbKsYrFAzLRq7DCyG4sE8lIg5mXqeXO0WLHJNvbW0lNjYWo9GIvi2DrnqV1qQy8vPzByXYNB3x4bEJZqzrb9YkU1JUeswqETEKCbky5Fanl+Uoj11I+1OXCHSHQK+Ah0mhFj4F1SPwuj/yLAEIj5JdHtGpCjo99JgElloXdRebMOakkTU/nvhsmU7u8wi6agQd1bLrIzpZ+kr7x8j92Fulm56jXbY3Ziz/yM3Q5/PR2tqK2WzGYrGQGJdMZEcOPlMSYVE6sgp0JM0e/VC5q6ub93efIblnCVGRUWRfpYw5Ccb/2rZWSI9pfaS0LQ12czAalvYODv6pjqSehaTOCmfulpF37G63m5aWFsxmMz09PaSkpGA0GsfsjTMUtbW1bN68mT/84Q9yI3N5EVKDM5elYA88XHQ4HJhMJsxmM4qiYDQaycrKGlfqRedFeciYNBvmbAzuQ2iz2Th8+DALFizAaJRbVKtZcH63IDkfZn5m5Ha8fs9NFZzfo6J6Yd4W3Yi35CBDDGoP+MheqyNxZn9xF0IEvL3rPnDh6oIZ18HcuXMHvTbWZhXzcR+GJXoSsgd/SdjbVDprVNx2OSEYGS+tSfW9Ker+jg/V2+u+55Je1l6nbCUE6QkSlShb66KSFAQCewv0NKm4egRutwtTdzWL1szAODsRRSezI/1p6j6vzHpMmSvDCvq+pq5uQUuFPFAMi1ZIX6KQlDe08b8QctKx6m0bnS02PAktJCx0kJU7enmgzdTF+/9bT3pEPql5kcz49NAtfqPhdcnOo44LgsSZCrNuHntC0XA0N1h49w/NpIXlk7smnJk3jHxoORBVVQNf+m1tbURHRwe6Tsa6IWpoaGDjxo38+7//O+vWrRvrUwkFNMG+FEazRR3oX+0X75F8d/3Y2wRn/gKRibBgW3AWlRaLhRMnTvSLUfK6BGdLZMLI/OKhe3uHo/2MSuMhwczrguswaCqT3szzNg8fvWWxWDiyq5HMtFyiF7UHxuT7ensLIWgs8+GxQfYa/SA3OpCvvatLDt+4uuV0pDrE3JI+DPSRCuHRUoTCY6Wo6SNlHJetRWBtlskxAFGJCmqslQvmExSsXUl8fDyubkFHlUp3ozy4jMtSSJmjGzSF53HIA8Wui/JAMXWBTF7XDTMA4uqRk4o9DYKoZBmSG2tQsNlsmEwmmpubA6+N0WgM+J4LIag50sWJXa1kZ+Uyc10k6UvG17tsNcmQAY8Vsq+a2B7ohtMdvPfHNrLSZjJvYziGZZf2uEIIrFYrZrOZ5uZmhBCDXpvhMJlMbNiwgV//+tdcd911l7SO0fjNb35DaWkpBw8enOiH1gR7vPjr1YqiBOVh7Xa7A+LtdDr7dVUM/F2PXVD5EggvLLwzuHYnv2d3QUFBoM3On8vYUTX2xBCfW3Bmh0xrmX3z6Lty4ROc260Sl6WQvWbo0om/n3uGvhDFE0Hep3tr2n1eG4fDQXp6OulJmdjOxqPopCd2MCPKgQlG0VsG0fevv6o+gbNT9iTbWmX5BOQOPTZDR3ymQoe1lcrKSgoL16B2R9JZLbC1CXR62SOcPHuwnanPJWg7K+g4L2vgyfnSWnS41jvVJ4W9pUKeDWSsUEhdMPShov+1MZvN2O12kuPSsVemYKq0M29lNnNuihzX+LYQAnM5NL7f2ya6nmHDJ8bDhYNdlP+5nVnzclj82fAJfWw/fk94s9mM1WolNTU1UDrp20bb3NzMhg0b+MUvfsENN9ww4evoi8vl4itf+QpVVVWaYE/VQkZjvB7WfjweDy0tLZhMJnp6ejAYDGRmZpKcnIzw8ZGh0zZGbOOC3rrxhQu0tbWxevXqfhaq7ecEtW+pZBXoMI7gVz0UpiMqrR8K5m4c7E8xFFazoO5tH7nr9MRnD/75mpoaGhsbKSgowFKpx9YsyL958Dmz33DIZDLR0WwjumsmcZEJGPLjSJqpJzIhuB2gENLYyNUtcHZJoXZ1SbFUdLIMEmtQiE1XArf/jY2NVJ2/yHxjAfZGPW6b9MZOytORlDd44ET1CCwXZE6izwOJM2T5IyJu+PX1NAmaDqm4uiFxphwPH+nn+9J2zsuJXW20t3YSO68H4wppHzvaROFAPDbpW91dJ0iZqzDzBsZ05zUSqk9w8uUeKt9sZ+FVWSzeFjFh5ZURr9s7odvc3ExbWxtRUVEcOnSIG264gS9+8Yv89Kc/Zf369ZO+jqeffpoFCxbw4x//+GMv2JdFl4jP58Pr9RIWFjbuA5Dw8HCys7PJzs4OHD7V1tZy/PgJOJ+Hri2dJUUxxBlHPuRTVZWKigoA1qxZ029X4eqWSddxmQoZV4xtfW6r9AFJmq0E7WtsbRLowhRiB/jl+FsLe3p6WLt2LXq9nog4la46Ob49UAT7Gg6pqkprczv1J9r58IMWIo9GkZASS2p2HNHxYegjZEsdSMc41SPwuuQdiscmsyKBQDhuQq4/e3Fwl8y5ilqaKq0YYwrp7JY/k7lAIT5zcO1Z9Qk6q2Xii9cpfVUMS4aeUPTjsQmayuVhZUQ8zLpBN+QX21B4HYL6g4K64zYcahe3fHs28YbwflOoUVFRgdruSHXvrlrBxdfB54K86xXSxllKGQp3j+Do/1i5eLydVVuymHtzxIR0mATDwAldk8nEiy++yK233kpCQgIVFRXk5eUF3AonA4/Hw4EDB3j44Yf58Y9/PCnXCCVCWrAH2qJOlC+uXq8PfNCaylTOtdnR5zVz0lxHoiORrKysfi1xfjweD+Xl5aSlpQ3yQBGqLIWg9B4yjvFD03xC9isbrwj+96xmQUw6g2w9/Qnzq1evDqzRL2x+P4zh0Ol0ZGSmk5GZjtet0lzVg/liJ+dOtqJXI4mLiycuLi7w2ig6hbBICIuWHiURcXKSMzJ+6KlQn0fQ3aBy/rAZaztk58wjMVdHUt7g+jTIsk9nrRwP99jlKHnOVXpi0od/DqpPfvm1nJAHnhkr5C482IO3zmpB/TsCS2sXLkMdN925iIhIeRfln0JdtGhRoN/7yJEjQ9a9VZ9MGjeVC6JTFOYVjS9ncTi66wXH/9eOqb6VTz2URfaKS++QGi+KohAbG8vRo0f59a9/zac+9SleeeUVfvSjH/Hss8+SmDj0fMCl8sILL3D33XdPymOHIiFbEvEfLgohJmxycSAdVYILeyFljkzWANm7ajKZaG1tJS4uLmDC5Df2mTNnzqCWOADTEYHpsDquwQlnp6xFpy1UyCoI7kvJ4xCc3+0jY7mO1AXyd7xeL+Xl5aSmpg75hVL9po/IRIWcteO7S+npsWJqNNNsbgHAmGkgMzuTuLiRU1KEKuvX3Q0Cq0nQbGpBiXKz+OpcEnJ1QzrECVXQVSsjtzxWKXjpS3XEZoy8O+1pEjSVqbi65O4+syD4Lg6vQ9DwnsByXuDQWfDNuMhVN6wMqvQxsO6dGGXA82Eu9MRiWKqQe+3EOeEJIWg+CpW77bTbmrj+0RxScsfeETWRWK1WNmzYwFe/+lXuuuuuKbvu97//fY4fP46iKBw6dIgnnniCb3zjGxN5iZAqiYSkYI/1cHE82FsFp/8iE6bnbxv8YeqbHNPU1ITD4WD27Nnk5+cTERHR72dtzYJzu2QLX971Y78LqH1bxlAtKBp6Cm8ouusFDe/5yLtBT0yqgsvloqysjLy8vGGdz9rPqbSdUclZqyfWcGmvqdPpDAiUy+UKnAn4hy6EKnB0yJY9a5Msm+jCBC3OauKyBUsL5g15x+QX6rZKeUAZlayQvkRHXObIQu22yvSXrlpZ/sgq0I2pp7mzWpZAvE5QjU240xooXFMwrhJc62kfp/fY6OnpQT+/AcOicIzGsde9h8LnEVx8DWrL7HSH1XLjo7OJS5r8KcWRsNls3HbbbXzhC1/g85///LStY926dR/7GnbICfalHi4GQ6AjxAeL7hx5oMBsNnPmzBkWLVpEV1cXZrMZvV4fiP2KCIvibKlsb1uwbWwtfCCn7M7vUTEsUzBeEbzYt1SotJ8VzC/S4XDaOXz4MIsWLcJgMAz7O6pXUPeOis8tyLlKP67+4aHwH+g2NZjoNnuII4NoXyoR+hj0YbJTJjZTcLruCKlpKcydO3fQYwhfr1Cf7hXqJIW0JTris0YWatUnLVJbP5Rv1fSlCumLgy9/9N1Vx6QpeGdcxKFYWLVq1ZhLcD63oO5tmREZa5QOexHxBOreLS0tQde9h8LZKTj3MrRetOEwXOD6BxYQHT29Yu1wONiyZQt33HEHDzzwwLSuZZLQBHs4JuJwcTRUn+BsqewIWXDHYG/ovlRXV2MymSgoKOi3q/YP6phMJrorEojoNLJ0WwyGOcEFqPal5m8qNrMU3rGIff27PlzdkLq2m+PHj/frAx8Jt1VQ/64PoYJhiY74nOCHegYiVIGrm0DLnqNd4PMKnC47Dl07dn0rSTkRZGSmU1NTQ25uLjNn9k+GV33SaKn9jMBjkzvqtMWjC7UQgu56MB2Wjnpj7f6Aj2rVPjdkXAEd0WdxOO1cccUVYxbrvuPlmasha+3QwyoDI+OC7WnuvCiNyGxWG67ZlVxbtGxK/D9GwuVyUVRUxIYNG3jooYem3E97igipJxUSgj0ZtqjDXafmTWmwM3v98Kkx/qRrt9s94oe3q05wbo8HfY4Fd0YNLpcrYH862gcQxr+7Bqh+3YfN3U17/AkKCgqIjQ3+y8JjF5iOqIGk8IRchZh0eVA43GGp6hW4bbIrwdUNrk7ZtufrHZyJiFOITYfYDDmF6B9db25u5vjx4+j1+kA+odFoRK+E01ElsJyVXR/RaQppi3TEGUfvoHB2CpoOC6xNgqgkyCrUjann2OMQNLwrw3Nj0mSIbZXpFKqqBsJyg0UIQcsJaHind7z8luD9pf09zSaTCZvN1i981/+eE0IeXNb/HbyRPbjmnuTqG1ZNu1i73W62bt3KZz7zGR599NGPq1iDJtgDLjAFh4t+uusFZ0sFmYUKOVcNfR2v18uRI0dITExk/vz5w67H6xSc3i4Tt+cXyZY1v4NeU1MTdrs9UNdNSho6s7HubZXuBsGCrWPbXQOU/amNbmHmmjvnjss/RQbOyrFv/8ShopM90PoIadgkRK93tUv0S0v3j6ZHJUF0qmxDHMp61j+yv3jxYtLS0rBarTTUmGmssOFriScuOhFDfhxZyyKJMYwu1F6XoPm4FHldeO/wy/zgO3KEEHRWQ/070vfEuFIhfbng5IcVhIWFsXjx4jG9/zwOuQHorJauirNuGv94uc/no62tDbPZTHt7O4mJiRhSM7EezqDzgo7w7C56siq46po1E+KXcyl4PB7uvPNO1qxZw3e/+92Ps1iDJth9HnyUMfPJoPOiIDFvaHFwOp2UlZUxa9asUSOLLu5X6ayG+bcrQ7aY+YdRmpqa6O7uJi0tLZDZqCgKrm7B2Z0q6YsUMlcHv7sWQnDu3DnqXo9k0ZXZZBeEj/5Lo+CxyxxEf8K4zwOoQK+Bkz4CwqIUImJlvT8ibnQ3OX/I64oVK0hOTsbVJWg/J3uihYAogwdPopl2RxNer7ffmPzAv41Qe61Pj8vyRcpcBeOKsflNe+yy/NF5URCTrjDz0wqRSWJQsnmwdDfIwz+PHXLWyZLKRL1/hRCYq7o48WcHnSYnCSs60Oe1sXbtmklrjwsWr9fLvffey5IlS/jhD3/4cRdr0ARb4vV6eeyxx1i/fj3XX3/9oM6LqcYvMEuXLiUtLW3En+2okm56wU4z9s1stFgsJCcnE26egbc5gUXb9EHvyoQQgaGdyKrFJObpyFw5OeWjS8FisVBRUcHKlavQ2eKwnFexmuSQj0wrV/od9PYtDdjtdtLT0wNTqFaTrFM7O+V5Q1ahdPgLFiEEHeeh4T15MGxcpZCxXKZJ9k02D/rxVEHjB2A6DFGJsh30UjtuBtJ+Rnqx68Ih+eo26ro/JCsri7a2NlRVDdy5BVN2m0h8Ph/3338/M2fO5IknnvgkiDVogi1RVZW33nqLkpIS3n33XQoKCiguLuaGG26Y8lu+lpYWKisrWbVqFfHx8SP+rMcuOP0X2To2//axD8gIIWg1Wzj5ogtXTCsZBd5+gbvD4S/VJCUlMW/ePKpeUYlKUci5MrQEu6WlhVMnT7MgoxBbfQTubkFYlELyXIXk/NG7aPxTqPUXWmg5piPcnkxyRiz518aRnDe2riG3Ve6qu2qlz/WMaxWikpVByebB4uoWVL8qzZvSFsna91izGkdCqLLLpOkDiM+GpKtbqW48w9q1awMbmmDq3pOBqqo8+OCDpKSk8OSTT07qtUIMTbAH4vV6OXjwIKWlpbz99tssW7aMrVu3cvPNN4/LInUs1NTU0NDQQEFBwahfFEIILr4huxPmFytEB5lCM5CWD1XMRwRzNim4dd00NTXR0tJCTExMoF2wrz+J2+2mrKysX5dF7ds+fC6YfdPkdNOMh4unG6kqs2CMnodO6IlOUUiZKw81lWDb7Jwy+NZyVoBeEDWzB0dcA+0dbcTHx2M0GsnIyOj3+gxECEH7aWj8QBpDZRUq0llPp/RLNh/YsTIS7WcFtW8BAmZ+hjH5mweDxyZtfbtqZHklemkL56vO9hPrgQxV9/YHV4z0+owVVVX5xje+QUREBP/6r//6SRJr0AR7ZHw+Hx988AE7duzgzTffZOHChRQXF3PrrbcGZZEaLEIIKisrsdvtrFy5Mqg2ws6LgurXVbLX6shYMc5WOCE4u0MlPFYh/5b+b/yenh6amppobm4mPDw8MIhy4sQJFi5cSEbGR6YhLSdV2s8I5t2mQz9BJkLjwecRdNcLLpS1YmlwkJObRdKsMFLyg/dEAdne135Guun5PIPr1H0HmZqbm4mIiAh8ufXtmHB1Cer+Lg9U47MVZnxKCaSN9002z8nJCe759YnuijUq5K/vn14+EVhNgnM7ZD181k2gZsjot5HEeiBCCDo7OzGbzbS0tARen4yMjEva9Kiqyre//W3cbje/+93vPmliDZpgB4+qqhw5coTS0lJee+01Zs+eTXFxMRs3brykwxefz8fRo0eJjY1l4cKFQd9mq15B+1lIWzj+CKeeRsHF/SozPqUbNjAXZF5gdXU1NTU1xMXFkZub2y8t3WERXHzTh3GljpS5U/shEqrA3gqdNYLuepXWlnY8OhvLrp1Bymz9mL5AhBB01YD5qOynjs+S/dSjZWgO6mc2GInoyKLzw2gUHWSvVUhd2CcxvTfZfO7cuWRmZga1NluLoPoVcHZBZgFkrRlbEEAwtBwXXHxD2q3Oux2sytjFeij6vj7++K+x1r1VVeXxxx+no6OD3//+959EsQZNsMeHqqqcPHmSkpISXnnlFbKysiguLmbz5s0kJycH/Tgul4vDhw+Tk5NDXl7e5C14GGoPqFjNgoXbRk6T8dfVV69eTVhYGGazmaampkAog9GYSfvhGDw2yL9FN6G11KEQQuC0yNbIrnqB1y5b6zrUOsLT7ay8ZtGYP9DWZoG5XGBvk2ECmat1IxpTDUeXycWpvd201zlQknqYca2e3PyMQDulP9l84cKFI06C9n2uzUeh4V0Ij4FZt0BCzsS+vqpXtgQ2H4ekWTDnNmjrMnPhwgXWrFkzoYfwAz2s/R1LI9W9hRD8+Mc/pq6ujueee27SBtkOHTrEt771LXQ6HQUFBfzmN7+ZlOtcAppgXyr+ckZpaSl79+4lNTWVoqIitmzZQnp6+rC/19PTw5EjR0Yd4Z4svC7B6b+opM5XyCocXtzq6+upqamhsLBwUF29b/CArdVLWF0+htmxzL0hFn3YxO6AhCqwt8m7gp5GOYmo6BTiMiE+By60HCcmLnpMdykAzg6B+ZgspYTHQMYVOpJnj/2uRfUKmo8Jmo/LjoqcqxXi83y0tkrf8+7ubhITE7FYLCxbtiyov3lf3+qkfIW8Gxiyx/xScHUJzu0Ea5NMnMm9BkxmE1VVVaxdu3ZC688D6Vv3tlgsJCQkDKp7CyH4+c9/zqlTp3jhhRcu2f9kJMxmM0lJSURFRfG5z32Oxx9/nKVLl07a9caBJtgTiRCC8+fPU1JSwp49e4iNjaWoqIjbb78do9EYEJLW1lY+/PBDVq1aRUJCwrSstf2coPF9lTkbdEPabPqDEdrb2wM765Hwer1Ul7dT974bb0Q3mQUKOfkZpKSkjKvlSgiBxypLAbZmga1Z1nAVnUKsERKyFeKzFYTe189mNljcVnmg2FEl0IWBYYlMiRlLmrwfGd4gcHbIQICcqwb3ZXd3d1NWVkZCQgI2my0gTsN15HRelOUJ1Q2510L60onrrfbTVSNHzFWfzAxNmafQ1NQ0JWI9kIF170OHDuF2u3G73VRWVvLiiy9O6Xruu+8+vve977Fo0aIpu2YQaII9WQghuHjxIqWlpezevRu9Xs+WLVuw2+2Ul5fzv//7v5PedTIS1W+oeGyCeVt0g4dDhODkyZP4fD6WL18+pvJCV51KU5kPq9WOK7IVZ3QraTNiyMrJHBTd1Pd6Xge4uuSot7ND7qa9DvknD4tRiMtQiMuSAQl+N0OPx0NZWRk5OTlBd1l4ndKcqe2M9PxOXaBgWDp8nNdI+Nwyk7HtFITHQu6nFBKHGAX399WvXLmSxMTEgDj5TZj6BsuG6yOoPyjrydFp8mBxLAemwSCEoOkDqH8bolNhXpG8RlNTE9XV1axZs2ZKxXEoTp8+zRNPPMHbb79NXl4eGzZs4Pbbb58SAa2oqOAHP/gB+/btm/RrjRFNsKcCIQQNDQ185Stf4dSpU+Tl5bFp0ya2bt1KXl7etDT921vlpN7A1BP/IWhcXNy40zk8Dul013VR4POoOFwO7J4u7J4uYuKjSE5JJiEuAeHV4XWB1y5LCn7C4xRiUhWi02QyecQQsWD+SdBgD+58brmm1lNyaCV5tkLGirEZNPWlq1b2VXtskLYEsgoGR4gBdHV1cfToUVavXj1sX31PTw9ms5mGc+30lKcR6Utm9tWx5F8fOa4d/0h4XXIQxnIWUhdC/q2yf7uxsZGLFy+GhFgLIXjmmWfYt28fO3bswGaz8eqrr9Ld3c1DDz00qde2WCxs2bKFv/zlLxiNxkm91jjQBHsqEELwwAMPEB8fz7/+67/S1tbGyy+/zI4dO7BarQHxnjt37rRObLndbg4fPkx2dvaEHIKqXoGtFRxtAnePwGMHa7eNrs5urLYeImPCSTEkkpaZSHRSGJEJClGJjNrZ0dcXZKRzAv8a2s8KWk7KjMeEXAXjFaN3fgyHxy4tUDsuyCCD3E8pw+Zu+qcsRzPEEkLQWiFNlVTFQ/TyZnrCGnC73YGd91BhzWPF3iZb9pwdMOPTsttEUUJLrAH+67/+i+3bt7Nz584pvQv1er3cdttt/OQnP6GwsHDKrjsGNMGeKk6cOMHy5csH/Xtrayu7du2itLSUtrY2Nm3aRHFxMYsWLZpS8XY4HJSVlTFv3rygW80uBSFEv17vyMjIQC/zSEND/vLCaBauqk+mmLdUCDwOmQhuvGJor5Vg12s5KwdgVA9krFTIWDF8a11bWxsffvgha9asGVF0PHZBzf5e06aZvaZNsR+VfPyThFarNTAmP55zgfbT0nJVFw5zN0sLWICGhgZqampCRqz/+7//mz/96U/s2bMn0DY6Vbz44os88sgjLF68GIAnn3ySK6+8ckrXMAqaYIcSFouF3bt3U1paSmNjI7feeitbt25l2bJlk9p36hfB5cuXk5KSMmnXGQmr1YrJZMJsNqPT6QJBvH3Fzr9jHWlsX/UJOqpkqILHJr01Mq4YfhccDM5OQf3fBT1NMtR4xqdG3qE3Nzdz5swZ1qxZE1QgrtcJOddAxorhDxb9HRV9PWD8HRUjtbmpPjlsYyqTI+Zzt0BkwkdiXVtbS2FhYUiI9Ysvvsjvf/979u7dO6GDaR8jNMEOVbq6uti7dy87duygqqqKm266iW3bto0rfWQk/DvBYLxLpoq+oQyqqso0nYiIQHvhUDtWv1C3npRDLzFpvUI9SpzXSKg+QctxMB8TKHrIXtN/AGYo/F0WI/Uvq15Bw7vQfEyWVWavZ0w7fyEEFoulX95n4NCyr42AVXB+J3TXg3EVzLz+ozuC+vp66urqWLNmzaS2ygVLSUkJv/3tb9m3b9+0uwCGMJpgXw5YrVZeeeUVduzYQWVlJTfccANFRUVceeWVlzRE0NjYSFVVFYWFhdNuQj8cLpeL06dP09TURExMDEajkaysrH5p4JbzsvPDY+sV6hUKcaOkxIyG1STHyp0dguR8hZyrlVGdDIMpLzjaBdWvyUNfw3KFnHWXFojrLy35x+TDwsLIzMwkzmek7tUofC45bJO+5KNrhJpY7969myeffJJ9+/ZN2x3eZYIm2JcbDoeDN954g5KSEo4fP851111HcXEx69atG9OHr6qqiubmZgoKCkLidng4Ll68GIhGAz6q6XbbibXnoGs3EE40sQYdGcsvXai9TkHTIdlREhmvkHPN0K16A6mtraWxsZHCwsIh/w59DxZ1ETDrRkiaPfGfP5vNzuk3LVS/6UMf42FekULeEkPg7qmuro76+vqQEetXX32Vn/zkJ+zbt29UK2ENTbAva1wuF3/9618pKSnh8OHDXH311RQXF3PdddcNK8JCCE6dOoXL5RpXXuBU4Q9H6O7u7meI5XVKY6bWSpUuiw1XRDu+NLNMi8mSoQzjeU5+r+rG92WqumEpGFcrQe1+q6qqaG1tpaBg6GRzj11w8U3oujj4YHEi8bml5WpbJSTPhRk3eWjrlJOoDoeDqKgoXC4XV111VUh8Se/fv5/HH3+cV155ZVqmfS9DNMH+uODxeDhw4AAlJSW89957FBYWBjy9+/oXV1RUEB0dPeVdKGPBn2PpH9xRFAW3VfZQd1yQfdQJuXLgJSZdGRTKkJSUFNSBnB9np6D+oKCnQRBrkK16Q01/DsW5c+fo6uoa9myh78RizjowrJj4iUWQBlzndoCjTR5gZl/V/zrV1dXU1tYSHx9Pd3c3qampZGYOP8w02Rw4cIDHHnuMffv2TUlX0seEkPrAaoI9Qfg9vUtKSvj73//O8uXLWb9+Pb/97W/56U9/yvr166d7icOiqirHjh0jOlr6gjjapJ1oV6388yfNVkhfPHyXhhCCjo4OmpqaaG1tJT4+nqysrCFHwFWvoPmEPPxTdNKrOm1RcD4iQgjOnDmD3T50srnPI2g4CC0n5MTi7FsI+ktgrFjOCS7sBZ1eGjcNdF6sra2lqamJwsJC9Ho9qqrS3t6OyWSivb2dhIQEjEYjBoNhSnbeBw8e5Otf/zp79+4N2lpWA9AE++OPz+djz549PPzww+Tm5jJr1iyKi4tZv359yLVOeb1eysvLSU1JwxCRT9tpga1FOvGlzlNIW6iMqZQghKCrqytwIBcdHR3o9Xa2hNNwUIb+Js9RyLky+Mf23wEMl2xub5WlCYdFkHGFQs7VTPjEIkhDrPq/Q+P7EJcpW/aikvpfp6amBpPJFBDroZ6L/zVqaWkJuh9+vBw6dIgHH3yQXbt2jSm0QQPQBPvjT3V1NcXFxTz11FNcffXVlJeXU1JSwuuvv86cOXMoKipi06ZN02ZC5Uca+peR6JpNmMWAxw4RcZC2UCF5ztBj32Olp6eH+mozNX934THHk5IZw4Jb4kmfE3yHjBCCEydODJls3tcKNSwKZt380YDKRNMvFWYF5N04+EthNLEeCqvVGvCuBjAajWRmZo44qRksR44c4f777+fll18mPz//kh/vE4gm2B93bDYbDQ0NzJ8/v9+/q6pKRUVFwNM7JyeHoqKiMXt6TwR9fUFcpzNQvdKUKSFn/OEMAxGqoPUUmA4LhA8SF7rwGZpobjUjhAhKmPzlmqGSzd09guo3oKd+8qxQ/fQ0Cs6/DB6HTIUxLB98nYsXLwa6gMbb+ul0OgP2uW63m4yMDIxGI4mJiWOuw584cYJ7772X7du3D3ovagSNJtgaH3WO+D2909LSKC4uZsuWLZPeauX3BVmyZAlpaWmoPjHhSSq2ZmnUZG8TJOQo5KxT+pUO+gqTx+MhIyMj0OvtR1XVYZPNLedkxqLqgxnXQtriyTlYFELQfAxq90NEvEyFiR1ignMixHogHo+Hlhbp7d3T0xNU8ICfU6dOcdddd/HSSy9Nutvet771LcrLy1m5ciX/9m//NqnXmgY0wdboj7+drrS0lD179hAXFxfw9M7IyJhQIQrWF2S8eB299qenBeGx0qc6afbIYurxeALibbfbMRgMZGRkcO7cOTIyMvolm/tcgtoD0qcjNkMeLI7XVGo0fB7BxVeh9RQk58OcTQzy3AZZAmttbWX16tWTlswyMHC3b1fOwIPds2fPsm3bNv7nf/6HZcuWTcp6/Bw9epRnnnmGP/7xjzz00EN88YtfDPTvf0zQBFtjeIQQVFdXBzy9w8PD2bJlC8XFxWRnZ1+SePt9QVavXj3hh59CFbSfgaYyaSFrWArGVWOvg3u9XkwmE6dOnUJRFLKyssjKyiIlJQVrE1S/Du4eyCqEzMKJz1j047AIzr0MjlbZGph99dBfOlMh1gPxd+X4x+RjYmJQFIXc3FysVitFRUU8//zzrFy5ctLX8vTTT5OWlsYdd9wR8ON55JFHJv26U0hICfb0j11p9ENRFPLz8/ne977Hd7/7XRoaGigtLeWee+7B6/UGxHusnt59zZEm2j7T1tJb/mgVxGfJ8kd0yvgd+mpra1myZAlZWVm0trZSV1PP+y/VIRoySMmMY2lxHAk5k9fH7G/ZU3Sw4I7hpyOrqqpoa2ujoKBgSvuqFUUhJSWFlJSUwJj89u3bue+++7BYLNxzzz2jWuBOFJ2dnYE7oMTERE6dOjUl1/2kEpojd5fI5s2b+X//7/9N9zIuGf+u6Zvf/CZ/+9vf2L59OzExMXz5y1/mqquu4p/+6Z84d+4co9wl0dDQwLlz57jyyisnxeu4/bT03c67XsecTeMXa5fLxfvvv09+fj45OTnodDoSwg1EnF2Owb2cGatjiVtXx5Hzb3PkyBFMJhNer3fCnodQBXUHBGdLIToFlt43sli3t7dPuVgPRFEUEhISuOmmm4iIiOA//uM/mDVrFg888AD33HPPpF8/MTGR7u5uQJbbJqPMpvERH7uSSEVFBd/5zncoLCzkZz/72XQvZ9JobW1l586dlJaWYrFY2LhxI1u3bh0UiFtdXY3ZbB7Wb2Mi8DrlEMyltAEOTDYXQtByAhrekX7SeTdA8hz5+APjvmJiYsjKyiIjI2PcQyhuq+DCbuiqhYwr5PWG6+O+cOECFouF1atXh4TNQFNTExs3buTXv/411113XeDfvV7vpHuXHD16lN///vf8/ve/5+GHH+a+++4L1SCC8RJSJZGPnWB/6UtfYuPGjZSXl3+sBbsvFouFXbt2sWPHDhobG9mwYQNFRUU899xzzJs3j4cffnjK6qvjwW63U1ZWFuhacVsFNW9K7+rEPNmuN1ysWF/nPLPZTEREBFlZWWMaQulpkPVqr1P2cRuWDf8ZDTWxbm5uZsOGDfzyl7/k+uuvn5Y1PProoxw9epQVK1bwu9/9blrWMImElGB/rGrYZ86cIT09/RN3W5aSksL999/P/fffT1dXF7t27eKOO+4gMjKS6Ohojh49OuGe3hOF1WqlvLycZcuWkZKSguW8oPavoHph5qcV0peN3GHiLwkkJCQwf/58bDYbJpOJw4cPoyhKwBp2qFKQEAJzOdS+BZGJsOQOiM0Y/lrnz5+ns7MzZMTan5b0s5/9bNrEGvg4tvKFLJelYJvNZu68885+/+bP4HviiSc4c+bMNK1s+omJieGNN95g27ZtPP7447z66qv86le/4vTp09x4440BT+9QEJy+yeaxUQlUvy5ku55BYdYtjKsWHhsby5w5c5gzZw4OhwOz2cyxY8fwer0B8Y6LixvksjdnIyOmuPvFOlS++CwWC5s2beJHP/oRt9xyy3QvR2OK+FiVRG6++WYURcFisdDe3s5zzz3HtddeO93LmlKsVisvv/wy9957b79/dzgcvP7665SUlFBRUcF1111HUVER11xzzbSUS/omm8fFxnHqf8DZKUNqJ6Ndz+12B3q9e5p9+E7MJlJNZM6NUWRfOfIuvq/lbCiIdWdnJxs2bOCxxx6juLh4upczLoQQIetcOYCQWuTHSrD9HDhwgP37939iathjxeVysX//fkpKSigvL+fqq69m69atXHvttVPiHDdUsnn7GUFkogzunUzaKgXn9/pwuKxErKjDE9eOwWAgMzOT5OTkQSJy9uxZenp6Qkase3p62LhxIw899NCgu8zLkcOHD2Oz2SgsLJzyAOAg0QRbI3TweDz87W9/o6SkhPfff5/CwkK2bt3K9ddfP2w+4qUQbLL5RKP65Ci7ubx/MK7P56O1tRWTyURnZycpKSlkZWWRmprK+fPnsVqtIRM6YbPZ2LRpE/fddx+f//znp3s546KyspLIyEjy8/PZtWsX27dvZ+3ataSkpHD33XdP9/KGQhNsjdDE6/XyzjvvBDy9r7jiCrZu3cpNN900IfmTwSabTzSubhmM29MoSy4zPj10yaWvZ3VjYyNhYWEsWbIEg8Ew7V02DoeDzZs389nPfpYHHnhgWtcyHoQQOBwOHn30Ub785S+zdOlS/uu//outW7fS2NjIhQsXuPPOO/F4PCGRzNMHTbA1Qh+fz8d7773Hjh07+Otf/8rixYspKipiw4YN47p1DSbZfDLoqhGc3yW7TvJvhdSFo3/+zpw5g81mIy8vD7PZTEtLC/Hx8WRmZk5Z4EBfnE4nxcXFbNy4ka9+9auXS+13SL7//e9TUFDA1q1bsdlsxMbGUl1dzXvvvUdXVxeLFy/u10seAoTUi60J9gTwhz/8geeeew6ARx55JFRv7caNqqocPnyYkpIS3njjDebMmRMQkGA8vYNJNp9ohBA0vicHb6JTYV4RRKeO/NkTQnD27NlAoo1fGIUQdHd3B0IZoqKiAoEDk/3l43a7AyWqRx555LIU6+rqan7xi1+QnJyMzWZj3bp1/ervJ06c4K677uKBBx7g29/+9jSudEhC6gXXBHsCqKmpIS8vD4/Hw9q1azly5Mh0L2nSUFWVEydOBDy9c3NzKS4u5rbbbhvS03u0ZPPJwOsUXNgDHRcgbRHMXj/6FKY/fszpdLJixYoRhdFqtQYGdfR6fUC8J7om7/F4uPPOO1m7di3f+c53LkuxBnnI/dJLL2G1Wvn3f/93bDYb3/3ud5k5cyY33ngjZrOZPXv2BEyjQqyDJGQWAppgTyhCCNauXcuhQ4emeylTgj+2y+/pbTAYKC4uZvPmzaSlpfEf//EfLF68eMpbB31uwan/AcNyyFg5uk+2EILTp0/jcrlGFeuBOBwOTCYTJpMJVVXJzMyckLQYr9fLvffey9KlS/nBD34QSgI2JgaK73/+53/y4YcfEhERQWNjI1/96lf59Kc/HfjvPp9v2s8LBhBSL7wm2BPIM888Q0dHBz/84Q+neylTjr+c4Pf0drlcREZG8n//93/MmDFjygVHqCLoYN/Tp0/jdrsDafHjxeVyYTabaWpqwu12BxJ14uPjx/S4Pp+P+++/n1mzZvGTn/zkshXrvviF+9lnn6W2tpYnnngCVVVDovtmFELqxb8sJx2ni+EmLF966SUOHTrEK6+8ws6dO6dncdOMoigsWLCAH/7wh9jtdiorKykoKODuu+8mIiKCoqIiioqKyMrKmhIBClasKysr8Xg8lyzWAJGRkcycOZOZM2fi8XgCXTH+UIbMzEySkpJGvI6qqjz44INkZWXxj//4j1Mm1pN9DuN/HsuWLaOyshLgchDrkEPbYU8AjY2NfPazn2X37t2kpKRM93KmlQMHDvDSSy/x9NNPo9PpEEJQX19PaWkpO3fuRFVVNm/ezNatW5k5c+a07R79Yu31eodMYZ9IvF4vra2tNDU10d3dHYj6Sk1N7XddVVX5+te/TlRUFL/61a+mVNCm6hzmnXfeoaqqivvuu29SHn8SCKkdtibYE8CDDz7IW2+9RXZ2NgCvvvrqlA6FhBrDHRoJITCZTLz88svs2LEDu93ObbfdxtatW5kzZ86Uibc/T9Pn8026WA9EVVXa2towmUxYLBaSk5Opq6vj+uuv5x/+4R/wer089dRT07b7/KSdwwSBJtgaGgAtLS0BT++Ojg42bdpEcXHxIE/vicQv1qqqsnTp0mmtDwshaGtr47HHHuPtt98mMjKSJ598kvXr11/yoeV4+SSfwwyDJtgaGgNpb28PeHqbTCY2bNjA1q1bWbJkyYTtNv1dLUKIaRfrvmv60Y9+RH19PQ8//DC7du3itdde45e//CU33njjhF9vtHOYn/3sZ+zcuTPUOjWmk+l/k/RBE2yNkKOzs5M9e/awY8cOLl68yPr16ykuLr4kAyYhBCdPnkRRFJYsWRIyYv3zn/+cyspK/vSnPwX61IUQqKo6paKpncMMy/S/UfqgCfZlzLe+9S3Ky8tZuXLlx9ZEvqenh3379rFjxw7OnDnDjTfeSHFxMWvXrg1avENVrP/5n/+Z8vJy/vznP0+7f4Z2DjMs0/9m6YMm2JcpR48e5ZlnnuGPf/wjDz30EF/84hcpKCiY7mVNKg6Hg9dee42SkhJOnjzJddddR3FxMevWrRt2NyqEoKKiAr1ez+LFi0NGrH/729/y9ttv85e//GVKvVU0xsz0v2H6oAn2ZcrTTz9NWload9xxB6WlpTQ2NgZGez8JOJ1O3nzzTUpLSykvL+eaa64JeHr7SwuqqlJRUUF4eDiLFi0KGbF++umnefXVVyktLQ06d1Jj2pj+N00ftM71y5TOzs6A8VJiYiKdnZ3Tu6ApJioqik2bNvH8889z9OhRtmzZwksvvcSKFSv40pe+xN69e9m2bRuHDh0KGbEGePbZZ9mzZw/bt2/XxFpjzGiTjpcpiYmJdHd3AzIb8ZMWPNyXiIgIbr75Zm6++Wa8Xi8HDhzg0UcfRVEUYmNjyczMnDBP70vh+eef56WXXmL37t1afVhjXGg77MuUK6+8kr/+9a8A7N+/n7Vr107zikIDnU7H//3f/7F582aOHz/Ol770Jd566y0KCwu5++67KSkpwW63T/m6/vznP/P888+zc+fOUI3C0rgM0GrYlzGPPvooR48eZcWKFfzud7+b7uWEBB6Ph+3bt3PXXXcNGvsuKyujtLSUN954g7lz5wY8vePj4yd1Tdu3b+epp55i7969JCYmTuq1NCac0Kil9aIJtsYnDlVVOX78OCUlJbz66qvMmDEj4Ok90aWlXbt28ctf/pJ9+/YN6ReuEfJogq2hESr4px9LSkrYt28fGRkZFBUVsWXLFlJTUy/psV955RV++tOf8sorr1zyY2lMG5pga2iEIv7UGb+nd2JiIkVFRdx+++0YDIYxdZq8+eab/PCHP2Tfvn0YDIZJXLXGJKMJtoZGqCOEoKqqipKSEnbv3k1UVBS33357UJ7eBw4c4Nvf/jb79u3DaDRO4ao1JgFNsDUubw4dOsS3vvUtdDodBQUF/OY3v5nuJU0qQghqa2vZsWMHO3fuRAjBli1bKC4uHuTp/c477/DII4+wd+/ewJi3xmWNJtgalzdms5mkpCSioqL43Oc+x+OPP87SpUune1lTgt/Te8eOHezYsQOn0xnw9G5vb+fBBx9k9+7dzJgxY8rXtnnzZpYuXcrPfvazKb/2x5iQEmxtcEZjzPS9zQ8PD/9EWXEqikJWVhZf//rX+drXvhbw9P7qV7/K8ePHKSsrmxaxrqiowOFwTPl1NaYWbXAmRPDf6Tz22GNUVFRM82qCo6KigtbWVhYtWjTdS5kWFEUhIyODBx98kP3791NbW8vs2bOnZS1PPfUUDz/88LRcW2Pq0AQ7RPDXQVNTU3n99ddRVZUXXniB7du3o6rqNK9uMBaLha9//es8++yz072UkGG6UmLOnDlDenr6J9qe4JOCVhIJEbxeL2FhYSxbtox/+Zd/QVEU3nnnHb7yla+g0+nw+XwoihISSdNer5d77rmHX/3qV1oXxBQyXFpMQkICTzzxBGfOnJmmlWlMFdqhYwjhdDp54IEHKCkp4cUXX+TWW28lKioKp9M57cZFfXnxxRd55JFHWLx4MQBPPvkkV1555TSv6pPLzTffjKIoWCwW2tvbee6557j22mune1kfF0Lq0FET7BCgtbWVgwcP8otf/ILNmzeza9cu3nnnHSIiIrBYLPzbv/0bXV1d/OQnP9FuezWG5cCBA+zfv1/rEplYNMHW6M++ffvYs2cPjzzyCJmZmdx///089dRTgW6Djo4O7rnnHvbt2zfNK9XQ+MQRUoI9/QVRDTZs2MB//ud/smjRIpKTk8nMzAxYpwIcPHiQ5cuXA4TkAaSGhsbUoB06hgA+n69fL/MzzzyD1+sFpF3ou+++y7Zt2wBCJjlFQ0Nj6tF22CFAX7H2l6jCwsJob2/na1/7Gn/6059oamoCNMEeyG9+8xvWrVs33cvQ0JgStB12iNFXkFNTU/nDH/5AZWUlNpttGlcVmrhcLo4fPz7dy9DQmDK0HXYI499tL1q0iIKCgmleTejx7LPP8oUvfGG6l6GhMWVogh3CaOWP4fF4PBw4cIDPfOYz070UDY0pQxNsjcuSF154gbvvvnu6l6GhMaVogq1xWXL27FmeeeYZbrnlFk6dOqWFEGt8ItAGZzQue9atW8fBgwenexkaH09Cqi6pCbaGhobG8ISUYGslEQ0NDY3LBE2wNTQ0NC4TNMHW0NDQuEzQBFtDQ0PjMmG00fSQKrhraGhofJLRdtgaGhoalwmaYGtoaGhcJmiCraGhoXGZoAm2hoaGxmWCJtgaGhoalwmaYGtoaGhcJvx/pkxtxtudHggAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "import jax\n", + "import jax.numpy as jnp\n", + "from evosax import CMA_ES\n", + "from evosax.problems import BBOBFitness\n", + "\n", + "# Instantiate the problem evaluator\n", + "rosenbrock = BBOBFitness(\"RosenbrockOriginal\", num_dims=2, seed_id=2)\n", + "rosenbrock.visualize(plot_log_fn=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -49,14 +78,6 @@ } ], "source": [ - "import jax\n", - "import jax.numpy as jnp\n", - "from evosax import CMA_ES\n", - "from evosax.problems import BBOBFitness\n", - "\n", - "# Instantiate the problem evaluator\n", - "rosenbrock = BBOBFitness(\"RosenbrockOriginal\", num_dims=2)\n", - "\n", "# Instantiate the search strategy\n", "rng = jax.random.PRNGKey(0)\n", "strategy = CMA_ES(popsize=20, num_dims=2, elite_ratio=0.5)\n", @@ -262,6 +283,11 @@ "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.7" + }, + "vscode": { + "interpreter": { + "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + } } }, "nbformat": 4, diff --git a/examples/07_brax_control.ipynb b/examples/07_brax_control.ipynb index dbd8ad9..f4cb91c 100644 --- a/examples/07_brax_control.ipynb +++ b/examples/07_brax_control.ipynb @@ -32,10 +32,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ + "import numpy as np\n", "from evojax.obs_norm import ObsNormalizer\n", "from evojax.sim_mgr import SimManager\n", "from evojax.task.brax_task import BraxTask\n", @@ -47,7 +48,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -67,14 +68,35 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "ParameterReshaper: 6248 parameters detected for optimization.\n" + "START EVOLVING 6248 PARAMS.\n", + "{'num_gens': 1} {'train_perf': 984.8515625, 'test_perf': 996.6640625}\n", + "{'num_gens': 50} {'train_perf': 977.61962890625, 'test_perf': 995.9266357421875}\n", + "{'num_gens': 100} {'train_perf': 972.63818359375, 'test_perf': 998.8201904296875}\n", + "{'num_gens': 150} {'train_perf': 974.7550048828125, 'test_perf': 1005.531982421875}\n", + "{'num_gens': 200} {'train_perf': 1276.9852294921875, 'test_perf': 1715.9610595703125}\n", + "{'num_gens': 250} {'train_perf': 1773.773681640625, 'test_perf': 2281.2216796875}\n", + "{'num_gens': 300} {'train_perf': 2309.93212890625, 'test_perf': 2937.6201171875}\n", + "{'num_gens': 350} {'train_perf': 2772.38134765625, 'test_perf': 3408.61474609375}\n", + "{'num_gens': 400} {'train_perf': 3173.67919921875, 'test_perf': 3793.0986328125}\n", + "{'num_gens': 450} {'train_perf': 3442.1396484375, 'test_perf': 4159.99365234375}\n", + "{'num_gens': 500} {'train_perf': 3810.6884765625, 'test_perf': 4592.73876953125}\n", + "{'num_gens': 550} {'train_perf': 4118.63671875, 'test_perf': 4821.9951171875}\n", + "{'num_gens': 600} {'train_perf': 4364.1015625, 'test_perf': 5058.63818359375}\n", + "{'num_gens': 650} {'train_perf': 4587.93603515625, 'test_perf': 5283.1171875}\n", + "{'num_gens': 700} {'train_perf': 4855.1455078125, 'test_perf': 5531.912109375}\n", + "{'num_gens': 750} {'train_perf': 5086.2080078125, 'test_perf': 5737.22265625}\n", + "{'num_gens': 800} {'train_perf': 5173.3076171875, 'test_perf': 5803.7421875}\n", + "{'num_gens': 850} {'train_perf': 5386.2861328125, 'test_perf': 6014.3095703125}\n", + "{'num_gens': 900} {'train_perf': 5541.9794921875, 'test_perf': 6128.41064453125}\n", + "{'num_gens': 950} {'train_perf': 5708.3310546875, 'test_perf': 6317.2353515625}\n", + "{'num_gens': 1000} {'train_perf': 5864.361328125, 'test_perf': 6467.5126953125}\n" ] } ], @@ -112,7 +134,7 @@ "\n", "print(f\"START EVOLVING {policy.num_params} PARAMS.\")\n", "# Run ES Loop.\n", - "for gen_counter in range(1500):\n", + "for gen_counter in range(1000):\n", " params = solver.ask()\n", " scores, _ = sim_mgr.eval_params(params=params, test=False)\n", " solver.tell(fitness=scores)\n", @@ -140,14 +162,14 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Cumulative reward: -2459.6707\n" + "Cumulative reward: 1117.1326\n" ] }, { @@ -171,11 +193,11 @@ " \n", " \n", " \n", "
\n", " \n", @@ -186,7 +208,7 @@ "" ] }, - "execution_count": 31, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -195,27 +217,34 @@ "from IPython.display import HTML\n", "from brax import envs\n", "from brax.io import html\n", + "import jax\n", "\n", "env = envs.create(env_name=\"ant\")\n", - "jit_env_reset = jax.jit(env.reset)\n", - "jit_env_step = jax.jit(env.step)\n", - "jit_inference_fn = jax.jit(network.apply)\n", + "task_reset_fn = jax.jit(env.reset)\n", + "policy_reset_fn = jax.jit(policy.reset)\n", + "step_fn = jax.jit(env.step)\n", + "act_fn = jax.jit(policy.get_actions)\n", + "obs_norm_fn = jax.jit(obs_normalizer.normalize_obs)\n", "\n", - "net_params = param_reshaper.reshape_single(state.mean)\n", + "best_params = solver.best_params\n", + "obs_params = sim_mgr.obs_params\n", "\n", + "total_reward = 0\n", "rollout = []\n", - "rng = jax.random.PRNGKey(seed=0)\n", - "env_state = jit_env_reset(rng=rng)\n", - "cum_reward = 0\n", - "for _ in range(1000):\n", - " rollout.append(env_state)\n", - " act_rng, rng = jax.random.split(rng)\n", - " norm_obs = evaluator.obs_normalizer.normalize_obs(env_state.obs, evaluator.obs_params)\n", - " act = jit_inference_fn(net_params, env_state.obs, act_rng)\n", - " env_state = jit_env_step(env_state, act)\n", - " cum_reward += env_state.reward\n", + "rng = jax.random.PRNGKey(seed=42)\n", + "task_state = task_reset_fn(rng=rng)\n", + "policy_state = policy_reset_fn(task_state)\n", + "while not task_state.done:\n", + " rollout.append(task_state)\n", + " task_state = task_state.replace(\n", + " obs=obs_norm_fn(task_state.obs[None, :], obs_params).reshape(1, 87))\n", + " act, policy_state = act_fn(task_state, best_params[None, :], policy_state)\n", + " task_state = task_state.replace(\n", + " obs=obs_norm_fn(task_state.obs[None, :], obs_params).reshape(87,))\n", + " task_state = step_fn(task_state, act[0])\n", + " total_reward = total_reward + task_state.reward\n", "\n", - "print(\"Cumulative reward:\", cum_reward)\n", + "print(\"Cumulative reward:\", total_reward)\n", "HTML(html.render(env.sys, [s.qp for s in rollout]))" ] }, @@ -229,9 +258,9 @@ ], "metadata": { "kernelspec": { - "display_name": "mle-toolbox", + "display_name": "snippets", "language": "python", - "name": "mle-toolbox" + "name": "snippets" }, "language_info": { "codemirror_mode": { @@ -243,7 +272,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.7" + "version": "3.8.11" } }, "nbformat": 4, From 8223aa43de68c03a6799d29967ad9fc051cf5127 Mon Sep 17 00:00:00 2001 From: RobertTLange Date: Mon, 5 Dec 2022 18:19:15 +0100 Subject: [PATCH 13/13] fix wdecay --- CHANGELOG.md | 1 + evosax/utils/reshape_fitness.py | 4 +--- examples/07_brax_control.ipynb | 38 ++++++++++++++++++++++++++++----- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3682c90..2ec91e2 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ - Fixed reward masking in `GymFitness`. Using `jnp.sum(dones) >= 1` for cumulative return computation zeros out the final timestep, which is wrong. That's why there were problems with sparse reward gym environments (e.g. Mountain Car). - Fixed PGPE sample indexing. +- Fixed weight decay. Falsely multiplied by -1 when maximization. ### [v0.0.9] - 15/06/2022 diff --git a/evosax/utils/reshape_fitness.py b/evosax/utils/reshape_fitness.py index 58c803d..800b3c8 100755 --- a/evosax/utils/reshape_fitness.py +++ b/evosax/utils/reshape_fitness.py @@ -42,12 +42,10 @@ def apply(self, x: chex.Array, fitness: chex.Array) -> chex.Array: if self.norm_range: fitness = range_norm_trafo(fitness, -1.0, 1.0) + # Apply wdecay after normalization - makes easier to tune # "Reduce" fitness based on L2 norm of parameters if self.w_decay > 0.0: l2_fit_red = self.w_decay * compute_l2_norm(x) - l2_fit_red = jax.lax.select( - self.maximize, -1 * l2_fit_red, l2_fit_red - ) fitness += l2_fit_red return fitness diff --git a/examples/07_brax_control.ipynb b/examples/07_brax_control.ipynb index f4cb91c..1902e31 100644 --- a/examples/07_brax_control.ipynb +++ b/examples/07_brax_control.ipynb @@ -153,6 +153,34 @@ " )" ] }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'num_gens': 1000} {'train_perf': 5864.361328125, 'test_perf': 6430.080078125}\n" + ] + } + ], + "source": [ + "test_scores, _ = sim_mgr.eval_params(\n", + " params=solver.best_params, test=True\n", + " )\n", + "print(\n", + " {\n", + " \"num_gens\": gen_counter + 1,\n", + " },\n", + " {\n", + " \"train_perf\": float(np.nanmean(scores)),\n", + " \"test_perf\": float(np.nanmean(test_scores)),\n", + " },\n", + ")" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -162,14 +190,14 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Cumulative reward: 1117.1326\n" + "Cumulative reward: 6469.828\n" ] }, { @@ -193,7 +221,7 @@ " \n", " \n", " \n", "
\n", "