From 13dde6de7581d8eb430559cf2cfa80f4e0f54aa2 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Wed, 17 Aug 2022 14:02:52 +0100 Subject: [PATCH 01/35] v0.10.4dev --- CHANGELOG.md | 3 +++ alibi_detect/version.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe940e466..e2918c71c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Change Log +## v0.10.4dev +[Full Changelog](https://github.com/SeldonIO/alibi-detect/compare/v0.10.3...master) + ## v0.10.3 ## [v0.10.3](https://github.com/SeldonIO/alibi-detect/tree/v0.10.3) (2022-08-17) [Full Changelog](https://github.com/SeldonIO/alibi-detect/compare/v0.10.2...v0.10.3) diff --git a/alibi_detect/version.py b/alibi_detect/version.py index 0447022c5..028aea0b5 100644 --- a/alibi_detect/version.py +++ b/alibi_detect/version.py @@ -2,7 +2,7 @@ # 1) we don't load dependencies by storing it in __init__.py # 2) we can import it in setup.py for the same reason # 3) we can import it into your module module -__version__ = "0.10.3" +__version__ = "0.10.4dev" # Define the config specification version. This is distinct to the library version above. It is only updated when # any detector config schema is updated, such that loading a previous config spec cannot be guaranteed to work. From 705b718bb71436f4fece9383156e4e77e9b8fe72 Mon Sep 17 00:00:00 2001 From: arnaudvl Date: Fri, 19 Aug 2022 16:56:18 +0100 Subject: [PATCH 02/35] Add KeOps MMD detector (#548) * first commit keops * update kernel and mmd keops * allow multiple kernel bandwidths for keops * fix bug * update mmd * remove learned kernel and base kernel_matrix MMD function * unify batched mmd2 * update keops mmd * update docs and kernel import * bugfixes * remove unused imports * add benchmarking example * update test mmd * add test mmd keops * update readme * bugfix kernel and update mmd test * remove print from test * update keops tests * Add save warning and update tests * Update setup and associated docs * Fix typing issue in * Install keops as part of CI * Add keops tox environment * Add keops to all dependency bucket * Fix minor issue * Protect GaussianRBF with import optional * Skip keops tests on Windows, and keops notebook test. Fix backend validator. * Skip keops kernel tests if not installed * Add pykeops to op deps ERROR_TYPES * Skip keops tests on MacOS * Add note to docs about linux-only support for keops * Add batch_size_permutations to pydantic models * remove print * remove unnecessary comment * change default bandwidth fn to None * update infer sigma * update test warning, update and clarify keops kernels logic * clean up * update docstring * fix bug * undo unnecessary kwarg removal * make test consistent with torch/tf backends * add _mmd2 test * remove unused import * clarify docs, remove redundant framework checks * remove print * update docs keops * batched version of sigma_mean part 1 * remove unused import * update keops kernels test Co-authored-by: Ashley Scillitoe Co-authored-by: Alex Athorne --- .github/workflows/ci.yml | 3 + README.md | 18 +- alibi_detect/cd/base.py | 5 - alibi_detect/cd/keops/__init__.py | 0 alibi_detect/cd/keops/mmd.py | 182 ++++++ alibi_detect/cd/keops/tests/test_mmd_keops.py | 120 ++++ alibi_detect/cd/mmd.py | 30 +- alibi_detect/cd/tests/test_mmd.py | 11 +- alibi_detect/saving/saving.py | 4 +- alibi_detect/saving/schemas.py | 4 +- alibi_detect/tests/test_dep_management.py | 47 +- alibi_detect/utils/frameworks.py | 9 +- alibi_detect/utils/keops/__init__.py | 8 + alibi_detect/utils/keops/kernels.py | 116 ++++ .../utils/keops/tests/test_kernels_keops.py | 66 ++ .../utils/missing_optional_dependency.py | 4 +- doc/source/cd/methods/mmddrift.ipynb | 63 +- doc/source/conf.py | 3 +- doc/source/examples/cd_mmd_keops.ipynb | 596 ++++++++++++++++++ doc/source/overview/getting_started.md | 33 +- examples/cd_mmd_keops.ipynb | 1 + setup.cfg | 12 + setup.py | 7 +- testing/test_notebooks.py | 1 + 24 files changed, 1277 insertions(+), 66 deletions(-) create mode 100644 alibi_detect/cd/keops/__init__.py create mode 100644 alibi_detect/cd/keops/mmd.py create mode 100644 alibi_detect/cd/keops/tests/test_mmd_keops.py create mode 100644 alibi_detect/utils/keops/__init__.py create mode 100644 alibi_detect/utils/keops/kernels.py create mode 100644 alibi_detect/utils/keops/tests/test_kernels_keops.py create mode 100644 doc/source/examples/cd_mmd_keops.ipynb create mode 120000 examples/cd_mmd_keops.ipynb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8cabf0089..05707f30a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,9 @@ jobs: if [ "$RUNNER_OS" != "Windows" ] && [ ${{ matrix.python }} < '3.10' ]; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. python -m pip install --upgrade --upgrade-strategy eager -e .[prophet] fi + if [ "$RUNNER_OS" == "Linux"]; then # Currently, we only support KeOps on Linux. + python -m pip install --upgrade --upgrade-strategy eager -e .[keops] + fi python -m pip install --upgrade --upgrade-strategy eager -e .[tensorflow,torch] python -m pip freeze diff --git a/README.md b/README.md index 996f39f65..df03a7be0 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ The package, `alibi-detect` can be installed from: pip install git+https://github.com/SeldonIO/alibi-detect.git ``` -- To install with the tensorflow backend: +- To install with the TensorFlow backend: ```bash pip install alibi-detect[tensorflow] ``` @@ -89,6 +89,11 @@ The package, `alibi-detect` can be installed from: pip install alibi-detect[torch] ``` +- To install with the KeOps backend: + ```bash + pip install alibi-detect[keops] + ``` + - To use the `Prophet` time series outlier detector: ```bash @@ -181,8 +186,8 @@ The following tables show the advised use cases for each algorithm. The column * #### TensorFlow and PyTorch support -The drift detectors support TensorFlow and PyTorch backends. Alibi Detect does not install these as default. See the -[installation options](#installation-and-usage) for more details. +The drift detectors support TensorFlow, PyTorch and (where applicable) [KeOps](https://www.kernel-operations.io/keops/index.html) backends. +However, Alibi Detect does not install these by default. See the [installation options](#installation-and-usage) for more details. ```python from alibi_detect.cd import MMDDrift @@ -198,6 +203,13 @@ cd = MMDDrift(x_ref, backend='pytorch', p_val=.05) preds = cd.predict(x) ``` +Or in KeOps: + +```python +cd = MMDDrift(x_ref, backend='keops', p_val=.05) +preds = cd.predict(x) +``` + #### Built-in preprocessing steps Alibi Detect also comes with various preprocessing steps such as randomly initialized encoders, pretrained text diff --git a/alibi_detect/cd/base.py b/alibi_detect/cd/base.py index 937a08c7d..c7518e985 100644 --- a/alibi_detect/cd/base.py +++ b/alibi_detect/cd/base.py @@ -602,11 +602,6 @@ def preprocess(self, x: Union[np.ndarray, list]) -> Tuple[np.ndarray, np.ndarray else: return self.x_ref, x # type: ignore[return-value] - @abstractmethod - def kernel_matrix(self, x: Union['torch.Tensor', 'tf.Tensor'], y: Union['torch.Tensor', 'tf.Tensor']) \ - -> Union['torch.Tensor', 'tf.Tensor']: - pass - @abstractmethod def score(self, x: Union[np.ndarray, list]) -> Tuple[float, float, float]: pass diff --git a/alibi_detect/cd/keops/__init__.py b/alibi_detect/cd/keops/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/alibi_detect/cd/keops/mmd.py b/alibi_detect/cd/keops/mmd.py new file mode 100644 index 000000000..86173ad13 --- /dev/null +++ b/alibi_detect/cd/keops/mmd.py @@ -0,0 +1,182 @@ +import logging +import numpy as np +from pykeops.torch import LazyTensor +import torch +from typing import Callable, Dict, List, Optional, Tuple, Union +from alibi_detect.cd.base import BaseMMDDrift +from alibi_detect.utils.keops.kernels import GaussianRBF +from alibi_detect.utils.pytorch import get_device + +logger = logging.getLogger(__name__) + + +class MMDDriftKeops(BaseMMDDrift): + def __init__( + self, + x_ref: Union[np.ndarray, list], + p_val: float = .05, + x_ref_preprocessed: bool = False, + preprocess_at_init: bool = True, + update_x_ref: Optional[Dict[str, int]] = None, + preprocess_fn: Optional[Callable] = None, + kernel: Callable = GaussianRBF, + sigma: Optional[np.ndarray] = None, + configure_kernel_from_x_ref: bool = True, + n_permutations: int = 100, + batch_size_permutations: int = 1000000, + device: Optional[str] = None, + input_shape: Optional[tuple] = None, + data_type: Optional[str] = None + ) -> None: + """ + Maximum Mean Discrepancy (MMD) data drift detector using a permutation test. + + Parameters + ---------- + x_ref + Data used as reference distribution. + p_val + p-value used for the significance of the permutation test. + x_ref_preprocessed + Whether the given reference data `x_ref` has been preprocessed yet. If `x_ref_preprocessed=True`, only + the test data `x` will be preprocessed at prediction time. If `x_ref_preprocessed=False`, the reference + data will also be preprocessed. + preprocess_at_init + Whether to preprocess the reference data when the detector is instantiated. Otherwise, the reference + data will be preprocessed at prediction time. Only applies if `x_ref_preprocessed=False`. + update_x_ref + Reference data can optionally be updated to the last n instances seen by the detector + or via reservoir sampling with size n. For the former, the parameter equals {'last': n} while + for reservoir sampling {'reservoir_sampling': n} is passed. + preprocess_fn + Function to preprocess the data before computing the data drift metrics. + kernel + Kernel used for the MMD computation, defaults to Gaussian RBF kernel. + sigma + Optionally set the GaussianRBF kernel bandwidth. Can also pass multiple bandwidth values as an array. + The kernel evaluation is then averaged over those bandwidths. + configure_kernel_from_x_ref + Whether to already configure the kernel bandwidth from the reference data. + n_permutations + Number of permutations used in the permutation test. + batch_size_permutations + KeOps computes the n_permutations of the MMD^2 statistics in chunks of batch_size_permutations. + device + Device type used. The default None tries to use the GPU and falls back on CPU if needed. + Can be specified by passing either 'cuda', 'gpu' or 'cpu'. + input_shape + Shape of input data. + data_type + Optionally specify the data type (tabular, image or time-series). Added to metadata. + """ + super().__init__( + x_ref=x_ref, + p_val=p_val, + x_ref_preprocessed=x_ref_preprocessed, + preprocess_at_init=preprocess_at_init, + update_x_ref=update_x_ref, + preprocess_fn=preprocess_fn, + sigma=sigma, + configure_kernel_from_x_ref=configure_kernel_from_x_ref, + n_permutations=n_permutations, + input_shape=input_shape, + data_type=data_type + ) + self.meta.update({'backend': 'keops'}) + + # set device + self.device = get_device(device) + + # initialize kernel + sigma = torch.from_numpy(sigma).to(self.device) if isinstance(sigma, # type: ignore[assignment] + np.ndarray) else None + self.kernel = kernel(sigma).to(self.device) if kernel == GaussianRBF else kernel + + # set the correct MMD^2 function based on the batch size for the permutations + self.batch_size = batch_size_permutations + self.n_batches = 1 + (n_permutations - 1) // batch_size_permutations + + # infer the kernel bandwidth from the reference data + if isinstance(sigma, torch.Tensor): + self.infer_sigma = False + elif self.infer_sigma: + x = torch.from_numpy(self.x_ref).to(self.device) + _ = self.kernel(LazyTensor(x[:, None, :]), LazyTensor(x[None, :, :]), infer_sigma=self.infer_sigma) + self.infer_sigma = False + else: + self.infer_sigma = True + + def _mmd2(self, x_all: torch.Tensor, perms: List[torch.Tensor], m: int, n: int) \ + -> Tuple[torch.Tensor, torch.Tensor]: + """ + Batched (across the permutations) MMD^2 computation for the original test statistic and the permutations. + + Parameters + ---------- + x_all + Concatenated reference and test instances. + perms + List with permutation vectors. + m + Number of reference instances. + n + Number of test instances. + + Returns + ------- + MMD^2 statistic for the original and permuted reference and test sets. + """ + k_xx, k_yy, k_xy = [], [], [] + for batch in range(self.n_batches): + i, j = batch * self.batch_size, (batch + 1) * self.batch_size + # construct stacked tensors with a batch of permutations for the reference set x and test set y + x = torch.cat([x_all[perm[:m]][None, :, :] for perm in perms[i:j]], 0) + y = torch.cat([x_all[perm[m:]][None, :, :] for perm in perms[i:j]], 0) + if batch == 0: + x = torch.cat([x_all[None, :m, :], x], 0) + y = torch.cat([x_all[None, m:, :], y], 0) + x, y = x.to(self.device), y.to(self.device) + + # batch-wise kernel matrix computation over the permutations + k_xy.append(self.kernel( + LazyTensor(x[:, :, None, :]), LazyTensor(y[:, None, :, :]), self.infer_sigma).sum(1).sum(1).squeeze(-1)) + k_xx.append(self.kernel( + LazyTensor(x[:, :, None, :]), LazyTensor(x[:, None, :, :])).sum(1).sum(1).squeeze(-1)) + k_yy.append(self.kernel( + LazyTensor(y[:, :, None, :]), LazyTensor(y[:, None, :, :])).sum(1).sum(1).squeeze(-1)) + c_xx, c_yy, c_xy = 1 / (m * (m - 1)), 1 / (n * (n - 1)), 2. / (m * n) + # Note that the MMD^2 estimates assume that the diagonal of the kernel matrix consists of 1's + stats = c_xx * (torch.cat(k_xx) - m) + c_yy * (torch.cat(k_yy) - n) - c_xy * torch.cat(k_xy) + return stats[0], stats[1:] + + def score(self, x: Union[np.ndarray, list]) -> Tuple[float, float, float]: + """ + Compute the p-value resulting from a permutation test using the maximum mean discrepancy + as a distance measure between the reference data and the data to be tested. + + Parameters + ---------- + x + Batch of instances. + + Returns + ------- + p-value obtained from the permutation test, the MMD^2 between the reference and test set, + and the MMD^2 threshold above which drift is flagged. + """ + x_ref, x = self.preprocess(x) + x_ref = torch.from_numpy(x_ref).float() # type: ignore[assignment] + x = torch.from_numpy(x).float() # type: ignore[assignment] + # compute kernel matrix, MMD^2 and apply permutation test + m, n = x_ref.shape[0], x.shape[0] + perms = [torch.randperm(m + n) for _ in range(self.n_permutations)] + # TODO - Rethink typings (related to https://github.com/SeldonIO/alibi-detect/issues/540) + x_all = torch.cat([x_ref, x], 0) # type: ignore[list-item] + mmd2, mmd2_permuted = self._mmd2(x_all, perms, m, n) + if self.device.type == 'cuda': + mmd2, mmd2_permuted = mmd2.cpu(), mmd2_permuted.cpu() + p_val = (mmd2 <= mmd2_permuted).float().mean() + # compute distance threshold + idx_threshold = int(self.p_val * len(mmd2_permuted)) + distance_threshold = torch.sort(mmd2_permuted, descending=True).values[idx_threshold] + return p_val.numpy().item(), mmd2.numpy().item(), distance_threshold.numpy() diff --git a/alibi_detect/cd/keops/tests/test_mmd_keops.py b/alibi_detect/cd/keops/tests/test_mmd_keops.py new file mode 100644 index 000000000..a64a78173 --- /dev/null +++ b/alibi_detect/cd/keops/tests/test_mmd_keops.py @@ -0,0 +1,120 @@ +from functools import partial +from itertools import product +import numpy as np +import pytest +import torch +import torch.nn as nn +from typing import Callable, List +from alibi_detect.utils.frameworks import has_keops +from alibi_detect.utils.pytorch import GaussianRBF, mmd2_from_kernel_matrix +from alibi_detect.cd.pytorch.preprocess import HiddenOutput, preprocess_drift +if has_keops: + from alibi_detect.cd.keops.mmd import MMDDriftKeops + +n, n_hidden, n_classes = 500, 10, 5 + + +class MyModel(nn.Module): + def __init__(self, n_features: int): + super().__init__() + self.dense1 = nn.Linear(n_features, 20) + self.dense2 = nn.Linear(20, 2) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = nn.ReLU()(self.dense1(x)) + return self.dense2(x) + + +# test List[Any] inputs to the detector +def preprocess_list(x: List[np.ndarray]) -> np.ndarray: + return np.concatenate(x, axis=0) + + +n_features = [10] +n_enc = [None, 3] +preprocess = [ + (None, None), + (preprocess_drift, {'model': HiddenOutput, 'layer': -1}), + (preprocess_list, None) +] +update_x_ref = [{'last': 750}, {'reservoir_sampling': 750}, None] +preprocess_at_init = [True, False] +n_permutations = [10] +batch_size_permutations = [10, 1000000] +configure_kernel_from_x_ref = [True, False] +tests_mmddrift = list(product(n_features, n_enc, preprocess, n_permutations, preprocess_at_init, update_x_ref, + batch_size_permutations, configure_kernel_from_x_ref)) +n_tests = len(tests_mmddrift) + + +@pytest.fixture +def mmd_params(request): + return tests_mmddrift[request.param] + + +@pytest.mark.skipif(not has_keops, reason='Skipping since pykeops is not installed.') +@pytest.mark.parametrize('mmd_params', list(range(n_tests)), indirect=True) +def test_mmd(mmd_params): + n_features, n_enc, preprocess, n_permutations, preprocess_at_init, update_x_ref, \ + batch_size_permutations, configure_kernel_from_x_ref = mmd_params + + np.random.seed(0) + torch.manual_seed(0) + + x_ref = np.random.randn(n * n_features).reshape(n, n_features).astype(np.float32) + preprocess_fn, preprocess_kwargs = preprocess + to_list = False + if hasattr(preprocess_fn, '__name__') and preprocess_fn.__name__ == 'preprocess_list': + if not preprocess_at_init: + return + to_list = True + x_ref = [_[None, :] for _ in x_ref] + elif isinstance(preprocess_fn, Callable) and 'layer' in list(preprocess_kwargs.keys()) \ + and preprocess_kwargs['model'].__name__ == 'HiddenOutput': + model = MyModel(n_features) + layer = preprocess_kwargs['layer'] + preprocess_fn = partial(preprocess_fn, model=HiddenOutput(model=model, layer=layer)) + else: + preprocess_fn = None + + cd = MMDDriftKeops( + x_ref=x_ref, + p_val=.05, + preprocess_at_init=preprocess_at_init if isinstance(preprocess_fn, Callable) else False, + update_x_ref=update_x_ref, + preprocess_fn=preprocess_fn, + configure_kernel_from_x_ref=configure_kernel_from_x_ref, + n_permutations=n_permutations, + batch_size_permutations=batch_size_permutations + ) + x = x_ref.copy() + preds = cd.predict(x, return_p_val=True) + assert preds['data']['is_drift'] == 0 and preds['data']['p_val'] >= cd.p_val + if isinstance(update_x_ref, dict): + k = list(update_x_ref.keys())[0] + assert cd.n == len(x) + len(x_ref) + assert cd.x_ref.shape[0] == min(update_x_ref[k], len(x) + len(x_ref)) + + x_h1 = np.random.randn(n * n_features).reshape(n, n_features).astype(np.float32) + if to_list: + x_h1 = [_[None, :] for _ in x_h1] + preds = cd.predict(x_h1, return_p_val=True) + if preds['data']['is_drift'] == 1: + assert preds['data']['p_val'] < preds['data']['threshold'] == cd.p_val + assert preds['data']['distance'] > preds['data']['distance_threshold'] + else: + assert preds['data']['p_val'] >= preds['data']['threshold'] == cd.p_val + assert preds['data']['distance'] <= preds['data']['distance_threshold'] + + # ensure the keops MMD^2 estimate matches the pytorch implementation for the same kernel + if not isinstance(x_ref, list) and update_x_ref is None: + p_val, mmd2, distance_threshold = cd.score(x_h1) + kernel = GaussianRBF(sigma=cd.kernel.sigma) + if isinstance(preprocess_fn, Callable): + x_ref, x_h1 = cd.preprocess(x_h1) + x_ref = torch.from_numpy(x_ref).float() + x_h1 = torch.from_numpy(x_h1).float() + x_all = torch.cat([x_ref, x_h1], 0) + kernel_mat = kernel(x_all, x_all) + mmd2_torch = mmd2_from_kernel_matrix(kernel_mat, x_h1.shape[0]) + np.testing.assert_almost_equal(mmd2, mmd2_torch, decimal=6) diff --git a/alibi_detect/cd/mmd.py b/alibi_detect/cd/mmd.py index 4391f2ccd..b00c20449 100644 --- a/alibi_detect/cd/mmd.py +++ b/alibi_detect/cd/mmd.py @@ -1,7 +1,7 @@ import logging import numpy as np from typing import Callable, Dict, Optional, Union, Tuple -from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, BackendValidator +from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, has_keops, BackendValidator from alibi_detect.utils.warnings import deprecated_alias from alibi_detect.base import DriftConfigMixin @@ -11,6 +11,9 @@ if has_tensorflow: from alibi_detect.cd.tensorflow.mmd import MMDDriftTF +if has_keops and has_pytorch: + from alibi_detect.cd.keops.mmd import MMDDriftKeops + logger = logging.getLogger(__name__) @@ -29,6 +32,7 @@ def __init__( sigma: Optional[np.ndarray] = None, configure_kernel_from_x_ref: bool = True, n_permutations: int = 100, + batch_size_permutations: int = 1000000, device: Optional[str] = None, input_shape: Optional[tuple] = None, data_type: Optional[str] = None @@ -66,6 +70,9 @@ def __init__( Whether to already configure the kernel bandwidth from the reference data. n_permutations Number of permutations used in the permutation test. + batch_size_permutations + KeOps computes the n_permutations of the MMD^2 statistics in chunks of batch_size_permutations. + Only relevant for 'keops' backend. device Device type used. The default None tries to use the GPU and falls back on CPU if needed. Can be specified by passing either 'cuda', 'gpu' or 'cpu'. Only relevant for 'pytorch' backend. @@ -82,27 +89,34 @@ def __init__( backend = backend.lower() BackendValidator( backend_options={'tensorflow': ['tensorflow'], - 'pytorch': ['pytorch']}, + 'pytorch': ['pytorch'], + 'keops': ['keops']}, construct_name=self.__class__.__name__ ).verify_backend(backend) kwargs = locals() args = [kwargs['x_ref']] pop_kwargs = ['self', 'x_ref', 'backend', '__class__'] + if backend == 'tensorflow': + pop_kwargs += ['device', 'batch_size_permutations'] + detector = MMDDriftTF + elif backend == 'pytorch': + pop_kwargs += ['batch_size_permutations'] + detector = MMDDriftTorch + else: + detector = MMDDriftKeops [kwargs.pop(k, None) for k in pop_kwargs] if kernel is None: if backend == 'tensorflow': from alibi_detect.utils.tensorflow.kernels import GaussianRBF - else: + elif backend == 'pytorch': from alibi_detect.utils.pytorch.kernels import GaussianRBF # type: ignore + else: + from alibi_detect.utils.keops.kernels import GaussianRBF # type: ignore kwargs.update({'kernel': GaussianRBF}) - if backend == 'tensorflow' and has_tensorflow: - kwargs.pop('device', None) - self._detector = MMDDriftTF(*args, **kwargs) # type: ignore - else: - self._detector = MMDDriftTorch(*args, **kwargs) # type: ignore + self._detector = detector(*args, **kwargs) # type: ignore self.meta = self._detector.meta def predict(self, x: Union[np.ndarray, list], return_p_val: bool = True, return_distance: bool = True) \ diff --git a/alibi_detect/cd/tests/test_mmd.py b/alibi_detect/cd/tests/test_mmd.py index 33e776e14..c070dcaeb 100644 --- a/alibi_detect/cd/tests/test_mmd.py +++ b/alibi_detect/cd/tests/test_mmd.py @@ -3,10 +3,13 @@ from alibi_detect.cd import MMDDrift from alibi_detect.cd.pytorch.mmd import MMDDriftTorch from alibi_detect.cd.tensorflow.mmd import MMDDriftTF +from alibi_detect.utils.frameworks import has_keops +if has_keops: + from alibi_detect.cd.keops.mmd import MMDDriftKeops n, n_features = 100, 5 -tests_mmddrift = ['tensorflow', 'pytorch', 'PyToRcH', 'mxnet'] +tests_mmddrift = ['tensorflow', 'pytorch', 'keops', 'PyToRcH', 'mxnet'] n_tests = len(tests_mmddrift) @@ -18,16 +21,18 @@ def mmddrift_params(request): @pytest.mark.parametrize('mmddrift_params', list(range(n_tests)), indirect=True) def test_mmddrift(mmddrift_params): backend = mmddrift_params - x_ref = np.random.randn(*(n, n_features)) + x_ref = np.random.randn(*(n, n_features)).astype('float32') try: cd = MMDDrift(x_ref=x_ref, backend=backend) - except NotImplementedError: + except (NotImplementedError, ImportError): cd = None if backend.lower() == 'pytorch': assert isinstance(cd._detector, MMDDriftTorch) elif backend.lower() == 'tensorflow': assert isinstance(cd._detector, MMDDriftTF) + elif backend.lower() == 'keops' and has_keops: + assert isinstance(cd._detector, MMDDriftKeops) else: assert cd is None diff --git a/alibi_detect/saving/saving.py b/alibi_detect/saving/saving.py index 975fe2523..b1ae31983 100644 --- a/alibi_detect/saving/saving.py +++ b/alibi_detect/saving/saving.py @@ -46,8 +46,8 @@ def save_detector( if legacy: warnings.warn('The `legacy` option will be removed in a future version.', DeprecationWarning) - if 'backend' in list(detector.meta.keys()) and detector.meta['backend'] in ['pytorch', 'sklearn']: - raise NotImplementedError('Saving detectors with PyTorch or sklearn backend is not yet supported.') + if 'backend' in list(detector.meta.keys()) and detector.meta['backend'] in ['pytorch', 'sklearn', 'keops']: + raise NotImplementedError('Saving detectors with PyTorch, sklearn or keops backend is not yet supported.') # TODO: Replace .__args__ w/ typing.get_args() once Python 3.7 dropped (and remove type ignore below) detector_name = detector.__class__.__name__ diff --git a/alibi_detect/saving/schemas.py b/alibi_detect/saving/schemas.py index f5a8f9d33..6649fffb2 100644 --- a/alibi_detect/saving/schemas.py +++ b/alibi_detect/saving/schemas.py @@ -98,7 +98,7 @@ class DetectorConfig(CustomBaseModel): """ name: str "Name of the detector e.g. `MMDDrift`." - backend: Literal['tensorflow', 'pytorch', 'sklearn'] = 'tensorflow' + backend: Literal['tensorflow', 'pytorch', 'sklearn', 'keops'] = 'tensorflow' "The detector backend." meta: Optional[MetaData] = None "Config metadata. Should not be edited." @@ -634,6 +634,7 @@ class MMDDriftConfig(DriftDetectorConfig): sigma: Optional[NDArray[np.float32]] = None configure_kernel_from_x_ref: bool = True n_permutations: int = 100 + batch_size_permutations: int = 1000000 device: Optional[Literal['cpu', 'cuda']] = None @@ -652,6 +653,7 @@ class MMDDriftConfigResolved(DriftDetectorConfigResolved): sigma: Optional[NDArray[np.float32]] = None configure_kernel_from_x_ref: bool = True n_permutations: int = 100 + batch_size_permutations: int = 1000000 device: Optional[Literal['cpu', 'cuda']] = None diff --git a/alibi_detect/tests/test_dep_management.py b/alibi_detect/tests/test_dep_management.py index 4ee06fd8f..4333d93c9 100644 --- a/alibi_detect/tests/test_dep_management.py +++ b/alibi_detect/tests/test_dep_management.py @@ -66,8 +66,8 @@ def test_cd_torch_dependencies(opt_dep): dependency_map = defaultdict(lambda: ['default']) for dependency, relations in [ - ("HiddenOutput", ['torch']), - ("preprocess_drift", ['torch']) + ("HiddenOutput", ['torch', 'keops']), + ("preprocess_drift", ['torch', 'keops']) ]: dependency_map[dependency] = relations from alibi_detect.cd import pytorch as cd_pytorch @@ -156,8 +156,8 @@ def test_torch_model_dependencies(opt_dep): dependency_map = defaultdict(lambda: ['default']) for dependency, relations in [ - ("TransformerEmbedding", ['torch']), - ("trainer", ['torch']), + ("TransformerEmbedding", ['torch', 'keops']), + ("trainer", ['torch', 'keops']), ]: dependency_map[dependency] = relations from alibi_detect.models import pytorch as torch_models @@ -255,20 +255,33 @@ def test_torch_utils_dependencies(opt_dep): dependency_map = defaultdict(lambda: ['default']) for dependency, relations in [ - ("batch_compute_kernel_matrix", ['torch']), - ("mmd2", ['torch']), - ("mmd2_from_kernel_matrix", ['torch']), - ("squared_pairwise_distance", ['torch']), - ("GaussianRBF", ['torch']), - ("DeepKernel", ['torch']), - ("permed_lsdds", ['torch']), - ("predict_batch", ['torch']), - ("predict_batch_transformer", ['torch']), - ("quantile", ['torch']), - ("zero_diag", ['torch']), - ("TorchDataset", ['torch']), - ("get_device", ['torch']), + ("batch_compute_kernel_matrix", ['torch', 'keops']), + ("mmd2", ['torch', 'keops']), + ("mmd2_from_kernel_matrix", ['torch', 'keops']), + ("squared_pairwise_distance", ['torch', 'keops']), + ("GaussianRBF", ['torch', 'keops']), + ("DeepKernel", ['torch', 'keops']), + ("permed_lsdds", ['torch', 'keops']), + ("predict_batch", ['torch', 'keops']), + ("predict_batch_transformer", ['torch', 'keops']), + ("quantile", ['torch', 'keops']), + ("zero_diag", ['torch', 'keops']), + ("TorchDataset", ['torch', 'keops']), + ("get_device", ['torch', 'keops']), ]: dependency_map[dependency] = relations from alibi_detect.utils import pytorch as pytorch_utils check_correct_dependencies(pytorch_utils, dependency_map, opt_dep) + + +def test_keops_utils_dependencies(opt_dep): + """Tests that the keops utils module correctly protects against uninstalled optional dependencies. + """ + + dependency_map = defaultdict(lambda: ['default']) + for dependency, relations in [ + ("GaussianRBF", ['keops']), + ]: + dependency_map[dependency] = relations + from alibi_detect.utils import keops as keops_utils + check_correct_dependencies(keops_utils, dependency_map, opt_dep) diff --git a/alibi_detect/utils/frameworks.py b/alibi_detect/utils/frameworks.py index b1def72af..1c708eb43 100644 --- a/alibi_detect/utils/frameworks.py +++ b/alibi_detect/utils/frameworks.py @@ -14,12 +14,19 @@ except ImportError: has_pytorch = False +try: + import pykeops # noqa + import torch # noqa + has_keops = True +except ImportError: + has_keops = False # Map from backend name to boolean value indicating its presence HAS_BACKEND = { 'tensorflow': has_tensorflow, 'pytorch': has_pytorch, - 'sklearn': True + 'sklearn': True, + 'keops': has_keops, } diff --git a/alibi_detect/utils/keops/__init__.py b/alibi_detect/utils/keops/__init__.py new file mode 100644 index 000000000..9da9f5073 --- /dev/null +++ b/alibi_detect/utils/keops/__init__.py @@ -0,0 +1,8 @@ +from alibi_detect.utils.missing_optional_dependency import import_optional + + +GaussianRBF = import_optional('alibi_detect.utils.keops.kernels', names=['GaussianRBF']) + +__all__ = [ + "GaussianRBF" +] diff --git a/alibi_detect/utils/keops/kernels.py b/alibi_detect/utils/keops/kernels.py new file mode 100644 index 000000000..380ddadb9 --- /dev/null +++ b/alibi_detect/utils/keops/kernels.py @@ -0,0 +1,116 @@ +from pykeops.torch import LazyTensor +import torch +import torch.nn as nn +from typing import Callable, Optional + + +def sigma_mean(x: LazyTensor, y: LazyTensor, dist: LazyTensor, n_min: int = None) -> torch.Tensor: + """ + Set bandwidth to the mean distance between instances x and y. + + Parameters + ---------- + x + LazyTensor of instances with dimension [Nx, 1, features] or [batch_size, Nx, 1, features]. + The singleton dimension is necessary for broadcasting. + y + LazyTensor of instances with dimension [1, Ny, features] or [batch_size, 1, Ny, features]. + The singleton dimension is necessary for broadcasting. + dist + LazyTensor with dimensions [Nx, Ny] or [batch_size, Nx, Ny] containing the + pairwise distances between `x` and `y`. + n_min + In order to check whether x equals y after squeezing the singleton dimensions, we check if the + diagonal of the distance matrix (which is a lazy tensor from which the diagonal cannot be directly extracted) + consists of all zeros. We do this by computing the k-min distances and k-argmin indices over the + columns of the distance matrix. We then check if the distances on the diagonal of the distance matrix + are all zero or not. If they are all zero, then we do not use these distances (zeros) when computing + the mean pairwise distance as bandwidth. The default `None` sets k to Nx (=Ny). If Nx becomes very large, + it is advised to set `n_min` to a lower value. + + Returns + ------- + The computed bandwidth, `sigma`. + """ + batched = len(dist.shape) == 3 + if not batched: + nx, ny = dist.shape + axis = 1 + else: + batch_size, nx, ny = dist.shape + axis = 2 + n_mean = nx * ny + if nx == ny: + n_min = n_min if isinstance(n_min, int) else nx + d_min, id_min = dist.Kmin_argKmin(n_min, axis=axis) + if batched: + d_min, id_min = d_min[0], id_min[0] # first instance in permutation test contains the original data + rows, cols = torch.where(id_min.cpu() == torch.arange(nx)[:, None]) + if (d_min[rows, cols] == 0.).all(): + n_mean = nx * (nx - 1) + dist_sum = dist.sum(1).sum(1)[0] if batched else dist.sum(1).sum().unsqueeze(-1) + sigma = (.5 * dist_sum / n_mean) ** .5 + return sigma + + +class GaussianRBF(nn.Module): + def __init__( + self, + sigma: Optional[torch.Tensor] = None, + init_sigma_fn: Callable = None, + trainable: bool = False + ) -> None: + """ + Gaussian RBF kernel: k(x,y) = exp(-(1/(2*sigma^2)||x-y||^2). A forward pass takes + a batch of instances x and y and returns the kernel matrix. + x can be of shape [Nx, 1, features] or [batch_size, Nx, 1, features]. + y can be of shape [1, Ny, features] or [batch_size, 1, Ny, features]. + The returned kernel matrix can be of shape [Nx, Ny] or [batch_size, Nx, Ny]. + x, y and the returned kernel matrix are all lazy tensors. + + Parameters + ---------- + sigma + Bandwidth used for the kernel. Needn't be specified if being inferred or trained. + Can pass multiple values to eval kernel with and then average. + init_sigma_fn + Function used to compute the bandwidth `sigma`. Used when `sigma` is to be inferred. + The function's signature should match :py:func:`~alibi_detect.utils.keops.kernels.sigma_mean`, + meaning that it should take in the lazy tensors `x`, `y` and `dist` and return a tensor `sigma`. + trainable + Whether or not to track gradients w.r.t. `sigma` to allow it to be trained. + """ + super().__init__() + init_sigma_fn = sigma_mean if init_sigma_fn is None else init_sigma_fn + if sigma is None: + self.log_sigma = nn.Parameter(torch.empty(1), requires_grad=trainable) + self.init_required = True + else: + sigma = sigma.reshape(-1) # [Ns,] + self.log_sigma = nn.Parameter(sigma.log(), requires_grad=trainable) + self.init_required = False + self.init_sigma_fn = init_sigma_fn + self.trainable = trainable + + @property + def sigma(self) -> torch.Tensor: + return self.log_sigma.exp() + + def forward(self, x: LazyTensor, y: LazyTensor, infer_sigma: bool = False) -> LazyTensor: + + dist = ((x - y) ** 2).sum(-1) + + if infer_sigma or self.init_required: + if self.trainable and infer_sigma: + raise ValueError("Gradients cannot be computed w.r.t. an inferred sigma value") + sigma = self.init_sigma_fn(x, y, dist) + with torch.no_grad(): + self.log_sigma.copy_(sigma.log().clone()) + self.init_required = False + + gamma = 1. / (2. * self.sigma ** 2) + gamma = LazyTensor(gamma[None, None, :]) if len(dist.shape) == 2 else LazyTensor(gamma[None, None, None, :]) + kernel_mat = (- gamma * dist).exp() + if len(dist.shape) < len(gamma.shape): + kernel_mat = kernel_mat.sum(-1) / len(self.sigma) + return kernel_mat diff --git a/alibi_detect/utils/keops/tests/test_kernels_keops.py b/alibi_detect/utils/keops/tests/test_kernels_keops.py new file mode 100644 index 000000000..0c1489410 --- /dev/null +++ b/alibi_detect/utils/keops/tests/test_kernels_keops.py @@ -0,0 +1,66 @@ +from itertools import product +import numpy as np +from alibi_detect.utils.frameworks import has_keops +import pytest +import torch +if has_keops: + from pykeops.torch import LazyTensor + from alibi_detect.utils.keops import GaussianRBF + +sigma = [None, np.array([1.]), np.array([1., 2.])] +n_features = [5, 10] +n_instances = [(100, 100), (100, 75)] +batch_size = [None, 5] +trainable = [True, False] +tests_gk = list(product(sigma, n_features, n_instances, batch_size, trainable)) +n_tests_gk = len(tests_gk) + + +@pytest.fixture +def gaussian_kernel_params(request): + return tests_gk[request.param] + + +@pytest.mark.skipif(not has_keops, reason='Skipping since pykeops is not installed.') +@pytest.mark.parametrize('gaussian_kernel_params', list(range(n_tests_gk)), indirect=True) +def test_gaussian_kernel(gaussian_kernel_params): + sigma, n_features, n_instances, batch_size, trainable = gaussian_kernel_params + + print(sigma, n_features, n_instances, batch_size, trainable) + + xshape, yshape = (n_instances[0], n_features), (n_instances[1], n_features) + if batch_size: + xshape = (batch_size, ) + xshape + yshape = (batch_size, ) + yshape + sigma = sigma if sigma is None else torch.from_numpy(sigma).float() + x = torch.from_numpy(np.random.random(xshape)).float() + y = torch.from_numpy(np.random.random(yshape)).float() + if batch_size: + x_lazy, y_lazy = LazyTensor(x[:, :, None, :]), LazyTensor(y[:, None, :, :]) + x_lazy2 = LazyTensor(x[:, None, :, :]) + else: + x_lazy, y_lazy = LazyTensor(x[:, None, :]), LazyTensor(y[None, :, :]) + x_lazy2 = LazyTensor(x[None, :, :]) + + kernel = GaussianRBF(sigma=sigma, trainable=trainable) + infer_sigma = True if sigma is None else False + if trainable and infer_sigma: + with pytest.raises(ValueError): + kernel(x_lazy, y_lazy, infer_sigma=infer_sigma) + else: + k_xy = kernel(x_lazy, y_lazy, infer_sigma=infer_sigma) + k_xx = kernel(x_lazy, x_lazy2, infer_sigma=infer_sigma) + k_xy_shape = n_instances + k_xx_shape = (n_instances[0], n_instances[0]) + axis = 1 + if batch_size: + k_xy_shape = (batch_size, ) + k_xy_shape + k_xx_shape = (batch_size, ) + k_xx_shape + axis = 2 + assert k_xy.shape == k_xy_shape and k_xx.shape == k_xx_shape + k_xx_argmax = k_xx.argmax(axis=axis) + k_xx_min, k_xy_min = k_xx.min(axis=axis), k_xy.min(axis=axis) + if batch_size: + k_xx_argmax, k_xx_min, k_xy_min = k_xx_argmax[0], k_xx_min[0], k_xy_min[0] + assert (torch.arange(n_instances[0]) == k_xx_argmax.cpu().view(-1)).all() + assert (k_xx_min >= 0.).all() and (k_xy_min >= 0.).all() diff --git a/alibi_detect/utils/missing_optional_dependency.py b/alibi_detect/utils/missing_optional_dependency.py index af52409cc..d31d35fa7 100644 --- a/alibi_detect/utils/missing_optional_dependency.py +++ b/alibi_detect/utils/missing_optional_dependency.py @@ -34,7 +34,9 @@ "tensorflow_probability": 'tensorflow', "tensorflow": 'tensorflow', "torch": 'torch', - "pytorch": 'torch' + "pytorch": 'torch', + "keops": 'keops', + "pykeops": 'keops', } diff --git a/doc/source/cd/methods/mmddrift.ipynb b/doc/source/cd/methods/mmddrift.ipynb index 3dc2e1d99..2f68cb664 100644 --- a/doc/source/cd/methods/mmddrift.ipynb +++ b/doc/source/cd/methods/mmddrift.ipynb @@ -2,14 +2,22 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "[source](../../api/alibi_detect.cd.mmd.rst)" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "# Maximum Mean Discrepancy\n", "\n", @@ -30,7 +38,11 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "## Usage\n", "\n", @@ -44,7 +56,7 @@ "\n", "Keyword arguments:\n", "\n", - "* `backend`: Both **TensorFlow** and **PyTorch** implementations of the MMD detector as well as various preprocessing steps are available. Specify the backend (*tensorflow* or *pytorch*). Defaults to *tensorflow*.\n", + "* `backend`: **TensorFlow**, **PyTorch** and [**KeOps**](https://github.com/getkeops/keops) implementations of the MMD detector are available. Specify the backend (*tensorflow*, *pytorch* or *keops*). Defaults to *tensorflow*.\n", "\n", "* `p_val`: p-value used for significance of the permutation test.\n", "\n", @@ -56,11 +68,11 @@ "\n", "* `preprocess_fn`: Function to preprocess the data before computing the data drift metrics. Typically a dimensionality reduction technique.\n", "\n", - "* `kernel`: Kernel used when computing the MMD. Defaults to a Gaussian RBF kernel (`from alibi_detect.utils.pytorch import GaussianRBF` or `from alibi_detect.utils.tensorflow import GaussianRBF` dependent on the backend used).\n", + "* `kernel`: Kernel used when computing the MMD. Defaults to a Gaussian RBF kernel (`from alibi_detect.utils.pytorch import GaussianRBF`, `from alibi_detect.utils.tensorflow import GaussianRBF` or `from alibi_detect.utils.keops import GaussianRBF` dependent on the backend used). Note that for the KeOps backend, the diagonal entries of the kernel matrices `kernel(x_ref, x_ref)` and `kernel(x_test, x_test)` should be equal to 1. This is compliant with the default Gaussian RBF kernel.\n", "\n", "* `sigma`: Optional bandwidth for the kernel as a `np.ndarray`. We can also average over a number of different bandwidths, e.g. `np.array([.5, 1., 1.5])`.\n", "\n", - "* `configure_kernel_from_x_ref`: If `sigma` is not specified, the detector can infer it via a heuristic and set `sigma` to the median pairwise distance between 2 samples. If `configure_kernel_from_x_ref` is *True*, we can already set `sigma` at initialization of the detector by inferring it from `x_ref`, speeding up the prediction step. If set to *False*, `sigma` is computed separately for each test batch at prediction time.\n", + "* `configure_kernel_from_x_ref`: If `sigma` is not specified, the detector can infer it via a heuristic and set `sigma` to the median (*TensorFlow* and *PyTorch*) or the mean pairwise distance between 2 samples (*KeOps*) by default. If `configure_kernel_from_x_ref` is *True*, we can already set `sigma` at initialization of the detector by inferring it from `x_ref`, speeding up the prediction step. If set to *False*, `sigma` is computed separately for each test batch at prediction time.\n", "\n", "* `n_permutations`: Number of permutations used in the permutation test.\n", "\n", @@ -73,23 +85,22 @@ "\n", "* `device`: *cuda* or *gpu* to use the GPU and *cpu* for the CPU. If the device is not specified, the detector will try to leverage the GPU if possible and otherwise fall back on CPU.\n", "\n", + "Additional KeOps keyword arguments:\n", "\n", - "Initialized drift detector example:\n", + "* `batch_size_permutations`: KeOps computes the `n_permutations` of the MMD^2 statistics in chunks of `batch_size_permutations`. Defaults to 1,000,000.\n", + "\n", + "Initialized drift detector examples for each of the available backends:\n", "\n", "\n", "```python\n", "from alibi_detect.cd import MMDDrift\n", "\n", - "cd = MMDDrift(x_ref, backend='tensorflow', p_val=.05)\n", - "```\n", - "\n", - "The same detector in PyTorch:\n", - "\n", - "```python\n", - "cd = MMDDrift(x_ref, backend='pytorch', p_val=.05)\n", + "cd_tf = MMDDrift(x_ref, backend='tensorflow', p_val=.05)\n", + "cd_torch = MMDDrift(x_ref, backend='pytorch', p_val=.05)\n", + "cd_keops = MMDDrift(x_ref, backend='keops', p_val=.05)\n", "```\n", "\n", - "We can also easily add preprocessing functions for both frameworks. The following example uses a randomly initialized image encoder in PyTorch:\n", + "We can also easily add preprocessing functions for the *TensorFlow* and *PyTorch* frameworks. Note that we can also combine for instance a PyTorch preprocessing step with a KeOps detector. The following example uses a randomly initialized image encoder in PyTorch:\n", "\n", "```python\n", "from functools import partial\n", @@ -158,7 +169,11 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "### Detect Drift\n", "\n", @@ -184,7 +199,11 @@ }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, "source": [ "## Examples\n", "\n", @@ -196,6 +215,10 @@ "\n", "[Drift detection on CIFAR10](../../examples/cd_mmd_cifar10.ipynb)\n", "\n", + "### Tabular\n", + "\n", + "[Scaling up drift detection with KeOps](../../examples/cd_mmd_keops.ipynb)\n", + "\n", "### Text\n", "\n", "[Text drift detection on IMDB movie reviews](../../examples/cd_text_imdb.ipynb)" @@ -207,7 +230,7 @@ "hash": "ffba93b5284319fb7a107c8eacae647f441487dcc7e0323a4c0d3feb66ea8c5e" }, "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -221,9 +244,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.11" + "version": "3.7.6" } }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/doc/source/conf.py b/doc/source/conf.py index 98630dfad..cd4bf6319 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -110,7 +110,8 @@ "numba", "pydantic", "toml", - "catalogue" + "catalogue", + "pykeops" ] # Napoleon settings diff --git a/doc/source/examples/cd_mmd_keops.ipynb b/doc/source/examples/cd_mmd_keops.ipynb new file mode 100644 index 000000000..dac1556e8 --- /dev/null +++ b/doc/source/examples/cd_mmd_keops.ipynb @@ -0,0 +1,596 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "27a4394b", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "# Scaling up drift detection with KeOps\n", + "\n", + "## Introduction\n", + "\n", + "A number of convenient and powerful kernel-based drift detectors such as the MMD detector ([Gretton et al., 2012](https://jmlr.csail.mit.edu/papers/v13/gretton12a.html)) do not scale favourably with increasing dataset size $n$, leading to quadratic complexity $\\mathcal{O}(n^2)$ for naive implementations. As a result, we can quickly run into memory issues by having to store the $[N_\\text{ref} + N_\\text{test}, N_\\text{ref} + N_\\text{test}]$ kernel matrix (on the GPU if applicable) used for an efficient implementation of the permutation test. Note that $N_\\text{ref}$ is the reference data size and $N_\\text{test}$ the test data size.\n", + "\n", + "We can however drastically speed up and scale up kernel-based drift detectors to large dataset sizes by working with symbolic kernel matrices instead and leverage the [KeOps](https://www.kernel-operations.io/keops/index.html) library to do so. For the user of $\\texttt{Alibi Detect}$ the only thing that changes is the specification of the detector's backend:\n", + "\n", + "\n", + "```python\n", + "from alibi_detect.cd import MMDDrift\n", + "\n", + "detector_torch = MMDDrift(x_ref, backend='pytorch')\n", + "detector_keops = MMDDrift(x_ref, backend='keops')\n", + "```\n", + "\n", + "In this notebook we will run a few simple benchmarks to illustrate the speed and memory improvements from using KeOps over vanilla PyTorch on the GPU (1x RTX 2080 Ti).\n", + "\n", + "## Data\n", + "\n", + "We randomly sample points from the standard normal distribution and run the MMD detectors with PyTorch and KeOps backends for the following settings:\n", + "\n", + "- $N_\\text{ref}, N_\\text{test} = [2, 5, 10, 20, 50, 100]$ (batch sizes in '000s)\n", + "- $D = [2, 10, 50]$\n", + "\n", + "Where $D$ denotes the number of features.\n", + "\n", + "## Requirements\n", + "\n", + "The notebook requires [PyTorch](https://pytorch.org/) and KeOps to be installed. These are optional dependencies for $\\texttt{Alibi Detect}$ and can be installed using:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a0bf1719", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "!pip install alibi-detect[keops]" + ] + }, + { + "cell_type": "markdown", + "id": "7ff93d59", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "Before we start let’s fix the random seeds for reproducibility:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "2ba95f29", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import torch\n", + "\n", + "def set_seed(seed: int) -> None:\n", + " torch.manual_seed(seed)\n", + " torch.cuda.manual_seed(seed)\n", + " np.random.seed(seed)\n", + "\n", + "set_seed(2022)" + ] + }, + { + "cell_type": "markdown", + "id": "1910895a", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "\n", + "## Vanilla PyTorch vs. KeOps comparison\n", + "\n", + "### Experiments\n", + "\n", + "First we define some utility functions to run the experiments:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "a1c65254", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "from alibi_detect.cd import MMDDrift\n", + "import matplotlib.pyplot as plt\n", + "from scipy.stats import kstest\n", + "from timeit import default_timer as timer\n", + "\n", + "\n", + "def eval_detector(p_vals: np.ndarray, threshold: float, is_drift: bool, t_mean: float, t_std: float) -> dict:\n", + " \"\"\" In case of drifted data (ground truth) it returns the detector's power.\n", + " In case of no drift, it computes the false positive rate (FPR) and whether the p-values\n", + " are uniformly distributed U[0,1] which is checked via a KS test. \"\"\"\n", + " results = {'power': None, 'fpr': None, 'ks': None}\n", + " below_p_val_threshold = (p_vals <= threshold).mean()\n", + " if is_drift:\n", + " results['power'] = below_p_val_threshold\n", + " else:\n", + " results['fpr'] = below_p_val_threshold\n", + " stat_ks, p_val_ks = kstest(p_vals, 'uniform')\n", + " results['ks'] = {'p_val': p_val_ks, 'stat': stat_ks}\n", + " results['p_vals'] = p_vals\n", + " results['time'] = {'mean': t_mean, 'stdev': t_std}\n", + " return results\n", + "\n", + "\n", + "def experiment(backend: str, n_runs: int, n_ref: int, n_test: int, n_features: int, mu: float = 0.) -> dict:\n", + " \"\"\" Runs the experiment n_runs times, each time with newly sampled reference and test data.\n", + " Returns the p-values for each test as well as the mean and standard deviations of the runtimes. \"\"\"\n", + " p_vals, t_detect = [], []\n", + " for _ in range(n_runs):\n", + " # Sample reference and test data\n", + " x_ref = np.random.randn(*(n_ref, n_features)).astype(np.float32)\n", + " x_test = np.random.randn(*(n_test, n_features)).astype(np.float32) + mu\n", + "\n", + " # Initialise detector, make and log predictions\n", + " p_val = .05\n", + " dd = MMDDrift(x_ref, backend=backend, p_val=p_val, n_permutations=100)\n", + " start = timer()\n", + " pred = dd.predict(x_test)\n", + " end = timer()\n", + "\n", + " if _ > 0: # first run reserved for KeOps compilation\n", + " t_detect.append(end - start)\n", + " p_vals.append(pred['data']['p_val'])\n", + "\n", + " del dd, x_ref, x_test\n", + " torch.cuda.empty_cache()\n", + "\n", + " p_vals = np.array(p_vals)\n", + " t_mean, t_std = np.array(t_detect).mean(), np.array(t_detect).std()\n", + " results = eval_detector(p_vals, p_val, mu == 0., t_mean, t_std)\n", + " return results\n", + "\n", + "\n", + "def format_results(n_features: list, backends: list, max_batch_size: int = 1e10) -> dict:\n", + " T = {'batch_size': None, 'keops': None, 'pytorch': None}\n", + " T['batch_size'] = np.unique([experiments['keops'][_]['n_ref'] for _ in experiments['keops'].keys()])\n", + " T['batch_size'] = list(T['batch_size'][T['batch_size'] <= max_batch_size])\n", + " T['keops'] = {f: [] for f in n_features}\n", + " T['pytorch'] = {f: [] for f in n_features}\n", + "\n", + " for backend in backends:\n", + " for f in T[backend].keys():\n", + " for bs in T['batch_size']:\n", + " for k, v in experiments[backend].items():\n", + " if f == v['n_features'] and bs == v['n_ref']:\n", + " T[backend][f].append(results[backend][k]['time']['mean'])\n", + "\n", + " for k, v in T['keops'].items(): # apply padding\n", + " n_pad = len(v) - len(T['pytorch'][k])\n", + " T['pytorch'][k] += [np.nan for _ in range(n_pad)]\n", + " return T\n", + "\n", + "\n", + "def plot_absolute_time(results: dict, n_features: list, y_scale: str = 'linear',\n", + " detector: str = 'MMD', max_batch_size: int = 1e10):\n", + " T = format_results(n_features, ['keops', 'pytorch'], max_batch_size)\n", + " colors = ['b', 'g', 'r', 'c', 'm', 'y', 'b']\n", + " legend, n_c = [], 0\n", + " for f in n_features:\n", + " plt.plot(T['batch_size'], T['keops'][f], linestyle='solid', color=colors[n_c]);\n", + " legend.append(f'keops - {f}')\n", + " plt.plot(T['batch_size'], T['pytorch'][f], linestyle='dashed', color=colors[n_c]);\n", + " legend.append(f'pytorch - {f}')\n", + " n_c += 1\n", + " plt.title(f'{detector} drift detection time for 100 permutations')\n", + " plt.legend(legend, loc=(1.1,.1));\n", + " plt.xlabel('Batch size');\n", + " plt.ylabel('Time (s)');\n", + " plt.yscale(y_scale);\n", + " plt.show();\n", + "\n", + "\n", + "def plot_relative_time(results: dict, n_features: list, y_scale: str = 'linear',\n", + " detector: str = 'MMD', max_batch_size: int = 1e10):\n", + " T = format_results(n_features, ['keops', 'pytorch'], max_batch_size)\n", + " colors = ['b', 'g', 'r', 'c', 'm', 'y', 'b']\n", + " legend, n_c = [], 0\n", + " for f in n_features:\n", + " t_keops, t_torch = T['keops'][f], T['pytorch'][f]\n", + " ratio = [tt / tk for tt, tk in zip(t_torch, t_keops)]\n", + " plt.plot(T['batch_size'], ratio, linestyle='solid', color=colors[n_c]);\n", + " legend.append(f'pytorch/keops - {f}')\n", + " n_c += 1\n", + " plt.title(f'{detector} drift detection pytorch/keops time ratio for 100 permutations')\n", + " plt.legend(legend, loc=(1.1,.1));\n", + " plt.xlabel('Batch size');\n", + " plt.ylabel('time pytorch / keops');\n", + " plt.yscale(y_scale);\n", + " plt.show();" + ] + }, + { + "cell_type": "markdown", + "id": "43a4ee7e", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "As detailed earlier, we will compare the PyTorch with the KeOps implementation of the MMD detector for a variety of reference and test data batch sizes as well as different feature dimensions. Note that for the PyTorch implementation, the portion of the kernel matrix for the reference data itself can already be computed at initialisation of the detector. This computation will not be included when we record the detector's prediction time. Since use cases where $N_\\text{ref} \\gg N_\\text{test}$ are quite common, we will also test for this specific setting. The key reason is that we cannot amortise this computation for the KeOps detector since we are working with lazily evaluated symbolic matrices.\n", + "\n", + "#### $N_\\text{ref} = N_\\text{test}$\n", + "\n", + "Note that for KeOps we could further increase the number of instances in the reference and test sets (e.g. to 500,000) without running into memory issues." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "47268603", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "experiments = {\n", + " 'keops': {\n", + " 0: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 10, 'n_features': 2},\n", + " 1: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 10, 'n_features': 2},\n", + " 2: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 10, 'n_features': 2},\n", + " 3: {'n_ref': 20000, 'n_test': 20000, 'n_runs': 10, 'n_features': 2},\n", + " 4: {'n_ref': 50000, 'n_test': 50000, 'n_runs': 10, 'n_features': 2},\n", + " 5: {'n_ref': 100000, 'n_test': 100000, 'n_runs': 10, 'n_features': 2},\n", + " 6: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 10, 'n_features': 10},\n", + " 7: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 10, 'n_features': 10},\n", + " 8: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 10, 'n_features': 10},\n", + " 9: {'n_ref': 20000, 'n_test': 20000, 'n_runs': 10, 'n_features': 10},\n", + " 10: {'n_ref': 50000, 'n_test': 50000, 'n_runs': 10, 'n_features': 10},\n", + " 11: {'n_ref': 100000, 'n_test': 100000, 'n_runs': 10, 'n_features': 10},\n", + " 12: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 10, 'n_features': 50},\n", + " 13: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 10, 'n_features': 50},\n", + " 14: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 10, 'n_features': 50},\n", + " 15: {'n_ref': 20000, 'n_test': 20000, 'n_runs': 10, 'n_features': 50},\n", + " 16: {'n_ref': 50000, 'n_test': 50000, 'n_runs': 10, 'n_features': 50},\n", + " 17: {'n_ref': 100000, 'n_test': 100000, 'n_runs': 10, 'n_features': 50}\n", + " },\n", + " 'pytorch': { # runs OOM after 10k instances in ref and test sets\n", + " 0: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 10, 'n_features': 2},\n", + " 1: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 10, 'n_features': 2},\n", + " 2: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 10, 'n_features': 2},\n", + " 3: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 10, 'n_features': 10},\n", + " 4: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 10, 'n_features': 10},\n", + " 5: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 10, 'n_features': 10},\n", + " 6: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 10, 'n_features': 50},\n", + " 7: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 10, 'n_features': 50},\n", + " 8: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 10, 'n_features': 50}\n", + " }\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "d556296a", + "metadata": { + "scrolled": true, + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "backends = ['keops', 'pytorch']\n", + "results = {backend: {} for backend in backends}\n", + "\n", + "for backend in backends:\n", + " exps = experiments[backend]\n", + " for i, exp in exps.items():\n", + " results[backend][i] = experiment(\n", + " backend, exp['n_runs'], exp['n_ref'], exp['n_test'], exp['n_features']\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "93396443", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "Below we visualise the runtimes of the different experiments. We can make the following observations:\n", + "\n", + "- The relative **speed** improvements of KeOps over vanilla PyTorch increase with increasing batch size.\n", + "\n", + "- Due to the explicit kernel computation and storage, the PyTorch detector runs out-of-memory after a little over 10,000 instances in each of the reference and test sets while KeOps keeps **scaling** up without any issues.\n", + "\n", + "- The relative speed improvements decline with growing **feature dimension**. Note however that we would not recommend using a (untrained) MMD detector on very high-dimensional data in the first place.\n", + "\n", + "The plots show both the absolute and relative (PyTorch / KeOps) mean prediction times for the MMD drift detector for different feature dimensions $[2, 10, 50]$." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "5d854bfb", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAEWCAYAAAAQHy/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdeVyVVf7A8c9hRwEVQYIUrybLRcxcsrQ0DWUcRdMcydFwLbXyl2mbM9OijVMzU5jpVC5l5pSNaZmO2qZh6Vg52qLIMi7hkoigoriy3PP743mwK4Iscbks3/frdV/c+6zfe7lwvs855zlHaa0RQgghRMPi4uwAhBBCCFHzJAEQQgghGiBJAIQQQogGSBIAIYQQogGSBEAIIYRogCQBEEIIIRogSQDqCKWUVkq1u8b6BUqpp+1eP6CUylJKnVVKNa/C+SzmOd2qGnN1Md9D2xo4T6h5LtcaOFeQUuorpVSeUirR0ecTFaeU+qNS6g1nxyGEo0kCUIJSKkMpla+UCiix/HuzQLSYr5ear+8qsd3L5vKx5uuxSqkis2A5q5T6SSn1llIqvDrj1lpP1lr/2TynOzAHiNVa+2itT5SXQPwa5nvcWk3H2qyUus9+mfkeDlTH8UucK0Mp1dfuPIfMcxVV97lKMRHIAfy01o/+2oMppYKVUmuVUkftv6d26z2VUkuUUmeUUseUUtNLrI9RSqUppc4rpZKUUq1/bUzOoJSaqZR6pxLb91ZKHbFfprV+Xmt9X1n7CFFfSAJQup+A3xe/UEp1ABqVst3/gNF227kB8cD+Ett9rbX2AZoAfYELwE6lVHR1BFvKFWsQ4AXsqY7jC4doDaToKozEVUatjA34BBhWxm4zgTDzvH2AJ5RS/c3jBQAfAk8D/sAOYEVl46qs2lC7JESDprWWh90DyACeAv5rt+wl4E+ABizmsqXm8iygmbksDvgY2AqMNZeNBbaWcp51wKprxPE4kAkcBcab525nd+7XgQ3AOYykYikwGwg3l2ngLPAF8JX5+py57J5Szudqvp8c4ADwkLmPm7m+CfCmGdPP5rlcAStwESgyj51rbu9pHu+Q+RktALztzncX8ANwBiNh6g/8xTzORfNY/zC3tX/vTYBlQDZw0Pxdudh/1uZ5T2Ekcr8t4/P9J0ahecE81xOApcR73my+z23mNv8GmgPvmnH/t/j7YG4fCXwOnATSgfgyzr0UKADyzeP2NT+vuebv+6j53NPcvjdwBHgSOAb88xrfGzfsvqd2y49i1AgVv/4z8C/z+URgm926xubnEnmNv5E/ACnm5/wW4GW3Ps783eaan92NJfZ9EtgFXDLjzcD4vu/C+I6+iZHEfgzkARv55W+sN3CklHj6YnyH8s3P9izwo7l+HJBqHusAMKnE+7SZ258FQjCSpXfsjj8YI5nONb8T1hLnfsyM/TRG4uRlrgvA+DvPNb8TWzC/q/KQR214OD2A2vaw+2eSjlG4uZr/fFtzdQIwG1gEPGAuex+j5qAiCcB4IKuMGPpjFJrR5j+p5VydAJwGbsOoxfEqjsdcb8GuIDOXXd6/jHNOBtKAVhhXgUlcWRiuBhaa8bQAttv9I73qPQIvA2vNY/liFJ4vmOu6mfH3M+O/HrOwMf/B3lfiWPbvfRmwxjymBaMWZoJdHAXA/ebv7QGMgk9d63dt9/qKz82MZR9wA0bikWKery9GwbUMeMvctjFwGKOwcQM6YSRTUWWc+/Lvy3z9HPCN+dkGYhScfzbX9QYKgb9hJArepR3T3PaqBABoZi4Lslv2O2C3+fwV4PUSx0kGhl3jc0u2+678h1++e52A48At5u9gjLm9p92+P5j7etst+waj0L/e3P8781heGEnss3afRakJgPl8JnaFt7lsoPk7VMAdwHmg8zWOd/kY/JJQ9wPcMRLFfYCH3bm3YyQO/hiJxmRz3QsYia+7+ehJGd9FecjDGQ9pAijbPzGq9/th/FH/XMZ2y4DRSqmmGP9cPqrg8Y9i/MMoTTxGwZKstT6H8Q+ppDVa6/9orW1a64sVPOe1xANztdaHtdYnMf55AUaHNWAA8IjW+pzW+jhGAT+itAMppRTGVeU0rfVJrXUe8Lzd9hOAJVrrz834f9Zap5UXoNnUMQL4g9Y6T2udASQCCXabHdRaL9ZGO/7bQDBGwVJVb2mt92utT2Ncke7XWm/UWhcCKzEKKTCuejO01m9prQu11t8DHwDDK3ieUcBzWuvjWutsYFaJ92XDKAQvaa0vVPI9+Jg/T9stO42RRBWvP82V7NeX5h9235W/8EuT2URgodb6W611kdb6bYwr/Vvt9p1n7mv/PuZrrbO01j9jXCl/q7X+3vxur+aXz7nStNbrzd+h1lp/CXyGURhXxD3AevO7WoBRu+QN9Cjxfo6an8W/gZvM5QUY37/WWusCrfUWrbVMviJqDWmDK9s/MarO22AU8qXSWm9VSgViNBGs01pfMMq/cl2PUS1YmhBgp93rg6Vsc7giJ6mEkBLHtD9na4wrmEy79+ZyjRgCMfpM7LTbXmFcEYJx9behCjEGmHHYx3YQ47Msdqz4idb6vHl+H6ouy+75hVJeFx+7NXCLUirXbr0bxveoIkK4+n2F2L3O/hWJ3lnzpx9G80rx8zy79X4l9rFfX5qS35XiWFsDY5RS/2e33oMr30tp35uKfs6VppT6LfAsxtW8C8Z3c3cFd7/i96K1timlDlPGdw6jdqH4vb6Ikbx/Zn4PF2mt/1qFtyCEQ0gNQBm01gcx2pAHYHSQupZ3gEe5RqJQiqEYVzqlycQoJIuFlhZiJc5VEdc652GMq7gArXVT8+GntW5fRiw5GP+029tt30QbHSGLj3dDGXFc633lYFxV2fdQD6Xs2pnyVOdneBj40u79NtXGHQUPVHD/o1z9vo7ava5yrFrrUxi/3452izvySyfRPfbrlFKNMX4/1+pEWvK7UhzrYeAvJT6HRlrr9+xDqto7AYzq+Msdcs1aocCyjq2U8sSoiXkJowmkKUbyqUrbvhRX/F7M2q1WVOA7Z9ZSPaq1bovRj2C6UiqmvP2EqCmSAFzbBOBOsxr+WuZhNBV8da2NlFKuSqk2Sqn5GG2Ps8rY9H1grFIqSinVCOPq5dfKAq51L/37wMNKqZZKqWbAjOIVWutMjGrTRKWUn1LKRSl1g1LqDrtjt1RKeZjb24DFwMtKqRYASqnrlVK/Mbd/Exhn3nrmYq6LLC9Os1r/feAvSilf81a16RgJWFWU95lUxjogXCmVoJRyNx83K6WsFdz/PeAppVSg2Sv/GSr5vpRSXhh9BAA8zdfFlpnHb2Z+1vdj9EMAo4o9Wik1zNznGWBXOc0yD5nfFX+M2q/iuwYWA5OVUrcoQ2Ol1ECl1LWaEyrjf4CXeUx3jE6gnnbrswCLUqr4f5uHuT4bKDRrA2JLbN9cKdWkjPO9Dww0v6vuGIn+JYw+GteklIpTSrUzk4bTGB1cbRV9o0I4miQA12C2G+6owHYntdabrtG+110pdRaj5/hmjOrVm7XWpVZDaq0/xugF/gVGh6MvqhJ/CTOBt5VSuUqp+FLWLwY+BX7E6IBVstZjNMY/0+Ke36sw2jcx49sDHFNK5ZjLnjRj/0YpdQajJ3eE+f62Y3SWexnjH+OX/HKV9QrwO6XUKaXUvFLi/D+Mq8ADGJ0tlwNLKvYRXOUFjEIxVyn1WBWPARhXexgFywiMq8Zj/NJpryJmY9x+twujevo7c1llFN/RAEaHTvs29mcx7rY4iPF5v6i1/sSMPRvj9sG/YPxub6GM/h12lmMkhQfM4842j7UDI7n4h3msfRidM6uF2RfjQeANjKvwcxiddIutNH+eUEp9Z/5eHsYoyE8BIzE6pxYfLw0j+Tpgfg/smyrQWqcD9wLzMWqgBgGDtNb5FQg3DON7fxb4GnhNa51UuXcshOMo6ZMihKgMpVQGxp0aG50dixCi6qQGQAghhGiAJAEQQgghGiBpAhBCCCEaIKkBEEIIIRqgOjEQUEBAgLZYLM4OQwgh6pSdO3fmaK0Dy99SNER1IgGwWCzs2FHu3XhCCCHsKKVKG0VUCECaAIQQQogGSRIAIYQQogGSBEAIIYRogCQBEEIIIRogSQCEEEKIBkgSACGEEKIBkgRACCGEaIAcmgAopaYppfYopZKVUu8ppbyUUm2UUt8qpfYppVYUzyEvhBDCTlYWPPIIXLrk7EhEPeWwBEApdT3GPNxdtdbRgCvGHON/A17WWrfDmJ97gqNiEEKIOunIEejVCxYvhj17nB2NqKcc3QTgBngrpdyARkAmcCewylz/NjDEwTEIIUTdsX8/9OwJx47Bp59C587OjkjUUw5LALTWPwMvAYcwCv7TwE4gV2tdaG52BLi+tP2VUhOVUjuUUjuys7MdFaYQQtQeqanGlf+ZM/DFF3D77c6OSNRjjmwCaAbcBbQBQoDGQP+K7q+1XqS17qq17hoYKHNZCCHque+/Nwp/mw2+/BK6dHF2RKKec2QTQF/gJ611tta6APgQuA1oajYJALQEfnZgDEIIUft9/TX06QPe3vDVVxAd7eyIRAPgyATgEHCrUqqRUkoBMUAKkAT8ztxmDLDGgTEIIUTt9sUX0K8fBAbC1q0QFubsiEQD4cg+AN9idPb7DthtnmsR8CQwXSm1D2gOvOmoGIQQolZbvx4GDACLxbjyDw11dkSiAXErf5Oq01o/CzxbYvEBoJsjzyuEELXeypUwciR07Gj09m/e3NkRiQZGRgIUQoiatnQpjBgBt94KmzZJ4S+cQhIAIYSoSa++CuPGQUwMfPIJNGni7IhEAyUJgBBC1JS//Q2mTIHBg2HtWmjc2NkRiQZMEgAhhHA0reHpp2HGDKPqf9Uq8PJydlSigXNoJ0AhhGjwtIZHH4WXX4YJE2DhQnB1dXZUQkgNgBBCOExREUyaZBT+Dz8MixZJ4S9qDUkAhBDCEQoKYPRoY0a/P/0J5s4FF/mXK2oPaQIQQojqdukS3HMPrFkDL7xgtP0LUctIAiCEENXp/HkYOhQ++wzmzzd6/QtRC0kCIIQQ1eXMGYiLg//8B5YsMe73F6KWkgRACCGqw4kT0L8//PADvPcexMc7OyIhrkkSACGE+LWOHTNm9Nu7F1avNmoBhKjlJAEQQohf49Ah6NsXjh41ZveLiXF2REJUiCQAQghRVfv2GQX+6dNGp78ePZwdkRAVJgmAEEJUxZ49xpV/QQF88QV07uzsiISoFIeNSqGUilBK/WD3OKOUekQp5a+U+lwptdf82cxRMQghhEPs3Al33AFKwVdfSeEv6iSHJQBa63St9U1a65uALsB5YDUwA9iktQ4DNpmvhRCibvjPf+DOO8HHB7ZsgagoZ0ckRJXU1LiUMcB+rfVB4C7gbXP528CQGopBCCF+nY0bITYWrrvOKPxvuMHZEQlRZTWVAIwA3jOfB2mtM83nx4Cg0nZQSk1USu1QSu3Izs6uiRiFEKJs//43DBxoFPpffQWtWjk7IiF+FYcnAEopD2AwsLLkOq21BnRp+2mtF2mtu2qtuwYGBjo4SiGEuIYVK+Duu6FjR9i8GYJKvW4Rok6piRqA3wLfaa2zzNdZSqlgAPPn8RqIQQghqmbJEvj9741b/DZuBH9/Z0ckRLWoiQTg9/xS/Q+wFhhjPh8DrKmBGIQQovLmzYMJE4x2/48/Bj8/Z0ckRLVxaAKglGoM9AM+tFv8V6CfUmov0Nd8LYQQtcvzz8PUqTBkiDGtb6NGzo5IiGrl0IGAtNbngOYllp3AuCtACCFqH63hT3+CF16AkSNh6VJwd3d2VEJUOxkJUAghitlsMG2aUfV///3w+uvg6ursqIRwCEkAhBACoKgIJk40Ov1NmwaJicZIf0LUUzU1DoAQQtReBQUwapRR+D/zjBT+okGQGgAhRMN28SLExxsD/fz97/D4486OSIgaIQmAEKLhOncO7roLNm2CV1+FBx90dkRC1BhJAIQQDdPp08bQvl9/bfT0HzOm3F2EqE8kARBCNDw5OfCb38Du3cYwv7/7nbMjEqLGSQIghGhYMjOhb184cAA++ggGDHB2REI4hSQAQoiG4+BBiImBY8dgwwbo08fZEQnhNJIACCEahv/9z7jyz8szJvW59VZnRySEU0kCIISo/3bvhn79jJH+kpLgppucHZEQTicDAQkh6rcdO6B3b2NI36++ksJfCJMkAEKI+mvLFrjzTmjSxHgeGensiISoNSQBEELUT599ZtzqFxJiXPm3bevsiISoVSQBEELUPx99BIMGQXi4Ufi3bOnsiISodSQBEELUL8uXGwP7dOpkdPhr0cLZEQlRKzk0AVBKNVVKrVJKpSmlUpVS3ZVS/kqpz5VSe82fzRwZgxCiAVm8GO69F3r2hM8/h2by70WIsji6BuAV4BOtdSTQEUgFZgCbtNZhwCbztRBC/Dpz58LEidC/vzHIj6+vsyMSolZzWAKglGoC9ALeBNBa52utc4G7gLfNzd4GhjgqBiFEA6A1zJ4N06bBsGFG+7+3t7OjEqLWc2QNQBsgG3hLKfW9UuoNpVRjIEhrnWlucwwIKm1npdREpdQOpdSO7OxsB4YphKiztIYZM+DppyEhAf71L/DwcHZUQtQJjkwA3IDOwOta607AOUpU92utNaBL21lrvUhr3VVr3TUwMNCBYQoh6iSbDaZMgb//HSZPNqb0dZPBTYWoKEcmAEeAI1rrb83XqzASgiylVDCA+fO4A2MQQtRHhYUwfjy89ho89pjx00VuahKiMhz2F6O1PgYcVkpFmItigBRgLTDGXDYGWOOoGIQQ9VB+PowcCW+/DbNmGTUASjk7KiHqHEfXl/0f8K5SygM4AIzDSDreV0pNAA4C8Q6OQQhRX1y4AMOHw/r18NJL8Oijzo5IiDrLoQmA1voHoGspq2IceV4hRD109iwMHgybN8OCBTBpkrMjEqJOkx4zQojaLzcXBgyA7dth2TJjsB8hxK8iCYAQonbLzobYWNizB1auhKFDnR2REPWCJABCiNrr55+hXz/46SdYu9YY5U8IUS0kARBC1E4ZGRATA8ePwyefwB13ODsiIeoVSQCEELVPerpR+J8/D5s2Qbduzo5IiHpHEgAhRO2ya5dR7Q9Gj/8bb3RqOELUVzJ0lhCi9ti+HXr3Nsbz/+orKfyFcCBJAIQQtcOXXxrV/s2awZYtEBFR/j5CiCqTBEAI4XyffGL08G/Vyij8LRZnRyREvScJgBDCuT780Bjhz2o1agFCQpwdkRANgiQAQgjneecdiI+Hrl3hiy9Apv4WosZIAiCEcI6FC2H0aOP+/s8+g6ZNnR2REA2KJABCiJqXmAiTJxvj+69fDz4+zo5IiAZHEgAhRM3RGmbNgsceM6b1/fBD8PJydlRCNEgyEJAQomZoDY8/blz9jx0Lb7wBrq7OjkqIBsuhCYBSKgPIA4qAQq11V6WUP7ACsAAZQLzW+pQj4xBCOJnNBg89BAsWGD/nzQMXqYAUwplq4i+wj9b6Jq11V/P1DGCT1joM2GS+FkLUV4WFxhX/ggXw5JMwf74U/kLUAs74K7wLeNt8/jYwxAkxCCFqQn4+3HMP/POfMHs2vPACKOXsqIQQOD4B0MBnSqmdSqmJ5rIgrXWm+fwYEFTajkqpiUqpHUqpHdnZ2Q4OUwhR7S5cgCFDjI5+L78Mf/qTFP5C1CKO7gR4u9b6Z6VUC+BzpVSa/UqttVZK6dJ21FovAhYBdO3atdRthBC1VF4eDBpkTOizeDHcd5+zIxJClODQGgCt9c/mz+PAaqAbkKWUCgYwfx53ZAxCiBp26pQxne/WrfDuu1L4C1FLOSwBUEo1Vkr5Fj8HYoFkYC0wxtxsDLDGUTEIIWrY8ePQpw98/z188AH8/vfOjkgIUQZHNgEEAauV0ebnBizXWn+ilPov8L5SagJwEIh3YAxCiJpy5Aj07QuHDsG6dUYtgBCi1nJYAqC1PgB0LGX5CSDGUecVQjjBgQMQEwMnTsCnn0LPns6OSAhRjnITAKVUd+BeoCcQDFzAqMpfD7yjtT7t0AiFELVbaqpx5X/xojGjX9eu5e8jhHC6a/YBUEp9DNwHfAr0x0gAooCnAC9gjVJqsKODFELUUj/8AL16QVERfPmlFP5C1CHl1QAkaK1zSiw7C3xnPhKVUgEOiUwIUbt98w389rfg6wubNkFYmLMjEkJUwjVrAIoLf7NHv4v5PFwpNVgp5W6/jRCiAUlKMqr9AwJgyxYp/IWogyp6G+BXgJdS6nrgMyABWOqooIQQtdiGDTBgAFgsxkA/rVs7OyIhRBVUNAFQWuvzwN3Aa1rr4UB7x4UlhKiVVq0yhveNioLNmyE42NkRCSGqqMIJgHk3wCiM3v8AMpG3EA3J228bE/t062b09g+Q7j9C1GUVTQCmAn8AVmut9yil2gJJjgtLCFGrvPaaMaXvnXca9/k3aeLsiIQQv1KFBgLSWn+F0Q+g+PUB4GFHBSWEqEX+/nd48kkYPBhWrAAvL2dHJISoBuWNA7BYKdWhjHWNlVLjlVKjHBOaEMKptIZnnjEK/xEjjPZ/KfyFqDfKqwF4FXjaTAKSgWyMAYDCAD9gCfCuQyMUQtQ8reHRR+Hll2HCBFi4EFyl248Q9ck1EwCt9Q9AvFLKB+jKL0MBp2qt02sgPiFETSsqggcegMWL4eGHjSTAxaEzhwshnKCifQDOApsdG4oQwukKCozOfsuXwx//CLNngzGjpxBVsnPnzhZubm5vANE4cAp6USobkFxYWHhfly5djpdc6cjpgIUQdcmlS8ZtfmvWwPPPwx/+4OyIRD3g5ub2xnXXXWcNDAw85eLiop0dT0Nis9lUdnZ21LFjx94Arpq3R7IxIQScP2/08l+zBubNk8JfVKfowMDAM1L41zwXFxcdGBh4GqP25SqVqgFQSjUyRwQUQtQXZ85AXBz85z/w5pswfryzIxL1i4sU/s5jfvalXuxXqAZAKdVDKZUCpJmvOyqlXqvgvq5Kqe+VUuvM122UUt8qpfYppVYopTwq9jaEENXu5EljUp+vvzba/aXwF6LBqGgTwMvAb4ATAFrrH4FeFdx3KpBq9/pvwMta63bAKWBCBY8jhKhOWVnQuzfs2gUffmi0/wtRz6Snp3uEhYXV2rlrBg8e3MZisUSHhYW1Hz58uOXSpUs11uu2wn0AtNaHSywqKm8fpVRLYCDwhvlaAXcCq8xN3gaGVDQGIUQ1OXwYevaE/fth/XoYNMjZEQnRII0aNerkgQMHktPT0/dcvHhRzZ07t8Ym2ahoAnBYKdUD0Eopd6XUY1x5VV+WucATGLciADQHcrXWhebrI8D1pe2olJqolNqhlNqRnZ1dwTCFEOXat88o/LOy4LPPICbG2REJUSNSUlI8rFZr1JdfftmosLCQSZMmtYyOjraGh4dHvfjiiwEANpuNSZMmtQwLC2sfHh4etXjx4mYA69at8+3atWtE796921ksluiRI0eGFhUVUVhYyLBhwyzF28+aNatFZWK65557Tru4uODi4kLXrl3PHTlypMaaxSvaCXAy8ApGYf0z8Bnw0LV2UErFAce11juVUr0rG5jWehGwCKBr167SgUSI6pCSYrT55+dDUhJ07uzsiEQDMn48rZKTaVSdx4yO5vySJZSsob7Kjz/+6DlixIgblixZ8lP37t0vvPTSSwFNmjQpSk5OTr1w4YK6+eabIwcNGnTmm2++abR7927v1NTUPZmZmW7dunWzxsbGngXYvXt34++//z45PDw8v1evXmHLli1r1q5du0uZmZnue/fu3QOQk5NTpSEzL126pFasWNF8zpw55b6X6lLRgYByMKYCrozbgMFKqQEYwwf7YSQRTZVSbmYtQEuMhEII4WjffQexseDhAV9+Ce1rbbOoENXq5MmTbkOGDGm3atWq/V26dLkIsHHjRr+0tLRGa9eubQaQl5fnmpKS4rVlyxbf+Pj4k25ubrRq1arwlltuObt169ZGTZo0sXXo0OFcVFRUPkB8fPzJLVu2+MTFxZ05fPiw55gxY1oNGjTo9NChQ89UJcYxY8aE3nrrrWf79+9/tvre+bVVKAFQSrUB/g+w2O+jtb5qYAG7dX/AmEIYswbgMa31KKXUSuB3wL+AMcCaKsYuhKiobdvgt7+Fpk1h0yZo187ZEYkGqCJX6o7g6+tbFBISkp+UlORTnABorVViYuKhYcOGXVFgr1+/vsy5rlWJUTGVUgQGBhYlJyenrF692m/BggWBK1as8F+5cmVG8TaFhYVER0dHAfTv3z937ty5R0se99FHHw3Oyclx+/TTT/f/undaORXtA/ARkAHMBxLtHlXxJDBdKbUPo0/Am1U8jhCiIjZtgn79ICgItm6Vwl80OO7u7vrjjz/e/9577zVfsGCBP0C/fv1Ov/7664HFve537drleebMGZdevXrlrVq1yr+wsJCjR4+6bd++3adnz57nwGgCSEtL8ygqKmLVqlX+PXv2zMvMzHQrKipi7NixuS+88MLPu3fvvqKJw83NjbS0tJS0tLSU0gr/OXPmBHzxxRdNPvroowOuNTzhVkX7AFzUWs+r6km01psx5xLQWh8AulX1WEKISvj3v2H4cAgLg88/h+uuc3ZEQjiFn5+f7dNPP93Xu3fvcF9f36Jp06blZGRkeHbo0MGqtVb+/v4FGzZs2J+QkJC7bds2H6vV2l4ppWfNmnUkNDS0cNeuXURHR5+bPHlyaEZGhlePHj3OJCQk5G7fvt17woQJFpvNpgCee+65I5WJ64knnmgdHBx8qWvXrlaAuLi4Uy+99FKmIz6DkpTW5fevU0qNxJgC+DPgUvFyrfV3jgvtF127dtU7duyoiVMJUX+sWAH33gs33QSffALNmzs7IlHDlFI7tdZdnRnDjz/+mNGxY8ccZ8ZQHdatW+ebmJgYlJSUtM/ZsVTWjz/+GNCxY0dLyeUVrQHoACRg3MNffEufNl8LIWqbt96C++6D226DdevAz8/ZEQkhapmKJgDDgbZa63xHBiOEqAbz58PDDxs9/levhkbVeteVEA1SXFxcXlxcXJ6z46hOFe0EmAw0dWQgQohq8MILRuE/ZAisXSuFvxCiTBWtARjljYkAACAASURBVGgKpCml/suVfQDKvA1QCFGDtIannoLnn4eRI2HpUnB3d3ZUQoharKIJwLMOjUIIUXU2G0ybBvPmwf33w+uvQw3fTiSEqHsqOhLgl44ORAhRBUVFMHEiLFkCjzwCc+aAqrHJxIQQddg1+wAopbaaP/OUUmfsHnlKqSoNdyiEqCYFBTBqlFH4P/20FP5CVJMZM2ZU24AZ06dPD3nmmWeCqrr/tm3bvG+66abIdu3aXTE5UXUorxNgYwCtta/W2s/u4au1lvuKhHCWixdh2DDjXv+//Q2ee04KfyGqybx584Irs73NZqOoqMghsfj4+Nj++c9//rRv3749n3322d4//vGPrao64VBJ5SUAMgufELXNuXMQF2eM8vfqq/DEE86OSIhaKz093aNNmzbtBw8e3KZt27bt+/fv3zYvL89l7dq1vn379r2heLvVq1f79evX74YHH3zw+kuXLrlERkZGDR48uA3AzJkzg8LCwtqHhYW1f+6551oUH9disUQPHTrUEh4e3n7//v0eq1at8ouKirJGREREde/ePbz42Kmpqd7dunWLaNmyZYfZs2dXarrgG2+88VKHDh0uAVgslgJ/f//CzMzMivbfu6byDtJCKTW9rJVa6znVEYQQooJOn4aBA+Hrr42e/mPGODsiISqlWzciSi67+25OzphBdl4eLjExhJVcf++95Dz8MCcyM3G76y5usF+3fTvp5Z0zIyPDa+HChRmxsbHnhg8fbnnxxRcDZ86cmTV16tTQo0ePuoWEhBQuWbKk+bhx43JGjhx5eunSpS3S0tJSALZs2dJo+fLlzXfu3JmqtaZLly7WmJiYvICAgKJDhw55vvnmmz/FxMRkHD161G3KlCmWzZs3p0VGRuZnZWVdvkrft2+f17Zt29Jzc3NdrVZr9OOPP57t6elZ6QvspKSkRgUFBSoqKupS+VuXr7waAFfAB/At4yGEqCk5ORATA99+C//6lxT+QlTQddddlx8bG3sOICEh4cS2bdt8XFxciI+PP7F48WL/nJwc1++++85n+PDhp0vuu3nzZp8BAwbk+vn52Zo0aWIbOHDgqaSkJF+A4ODg/JiYmHPmdo27deuWFxkZmQ8QFBR0uU0gNjY219vbWwcHBxf6+/sXHDlypNJX8AcPHnQfN25c28WLF2dU16RB5QWRqbV+rlrOJISousxMY0a/ffvgo4+MWgAh6qBrXbH7+mK71vrgYAorcsVfUmnT+AI88MADJwYOHNjOy8tLDxo06JR7JcfOaNSoka38rcD+at/V1ZXCwsIrAlq2bFnT559/PgRg0aJFGb169Tpvv/7kyZMuv/3tb9s9++yzPxcnHNWhvBoA6VUkhLMdPAi9ekFGBnz8sRT+QlRSZmamx8aNGxsDvPvuu/49evQ4C0abelBQUEFiYmLwxIkTL09Y5ObmpounCe7Tp8/ZDRs2NM3Ly3M5c+aMy4YNG5r16dPnqiGBe/fufW779u2+aWlpHgD2TQDlGT16dG7xlMElC/+LFy+qgQMHthsxYsSJcePGnaraJ1C68hKAmOo8mRCikvbuhZ49ITvbmM63Tx9nRyREnWOxWC7Onz+/Rdu2bdvn5ua6PfbYY9nF60aMGHEiODg4v3PnzheLl40aNSrbarVGDR48uM3tt99+fuTIkSc6d+5s7dKlizUhISH7tttuu1DyHCEhIYXz5s3LGDp0aLuIiIiooUOHtq2O2JcsWdLsv//9r8/y5csDIiMjoyIjI6O2bdvmXR3HrtB0wFU6sFJewFeAJ0ZTwyqt9bNKqTbAv4DmwE4gobxJhmQ6YNEgJSdD377GYD+ffQadOjk7IlHHyHTARm/9uLi4sL179+4pbf3o0aNDO3XqdH7atGl1fsrispQ1HXBFJwOqikvAnVrrjsBNQH+l1K3A34CXtdbtgFPABAfGIETdtGMH3HGHMaTvV19J4S+EA7Rv396akpLiPXny5BPOjsUZquVewtJoo2rhrPnS3Xxo4E5gpLn8bWAm8Lqj4hCiztm6FQYMgObNYdMmaFstNYlCNEgRERH5ZV3979mzJ7Wm46lNHFkDgFLKVSn1A3Ac+BzYD+RqrQvNTY4A15ex70Sl1A6l1I7s7OzSNhGi/vn8c4iNhZAQ2LJFCn8hhMM4NAHQWhdprW8CWgLdgMhK7LtIa91Va901MDDQYTEKUWusWWOM8BcWZlT7t2zp7IiEEPWYQxOAYlrrXCAJ6A40VUoVNz20BH6uiRiEqNXee88Y2/+mmyApCVpUarRQIYSoNIclAEqpQKVUU/O5N9APSMVIBH5nbjYGWOOoGISoE954w5jV7/bbYeNG8Pd3dkRCiAbAkTUAwUCSUmoX8F/gc631OuBJYLpSah/GrYBvOjAGIWq3uXPh/vvhN7+BDRvAV0bYFqI6paene4SFhbV3dhxlef755wNDQ0OjlVJd7Cf5sdlsjB07tlVoaGh0eHh41NatWxtV97kdeRfALuCqe5e01gcw+gMI0XBpDc8/D089BXffDcuXg6ens6MSQtSwO+644+ywYcNO33nnnVdMkrRy5comBw4c8MrIyEhOSkpq/OCDD4bu2rUrrTrPXSN9AIQQdrSGP/zBKPwTEmDFCin8hagBKSkpHlarNerLL79sVFhYyKRJk1pGR0dbw8PDo1588cUAMK68J02a1DIsLKx9eHh41OLFi5sBrFu3zrdr164RvXv3bmexWKJHjhwZWlRURGFhIcOGDbMUbz9r1qxKdeC57bbbLkRERFw1GN6aNWuajho16oSLiwsxMTHnzpw543bw4MHKTVZQDofVAAghSmGzwcMPw6uvwuTJxk8XycNFwzB+zfhWyceTq7UqO7pF9Pkldy05XN52P/74o+eIESNuWLJkyU/du3e/8NJLLwU0adKkKDk5OfXChQvq5ptvjhw0aNCZb775ptHu3bu9U1NT92RmZrp169bNGhsbexZg9+7djb///vvk8PDw/F69eoUtW7asWbt27S5lZma6F481kJOTUy1T9WVmZrpbLJbLiUFwcHD+wYMH3Vu3bl1QHccHqQEQouYUFsKECUah/+ij8NprUvgLUQNOnjzpNmTIkHbvvPPOge7du18A2Lhxo9/777/fPDIyMqpTp07WU6dOuaWkpHht2bLFNz4+/qSbmxutWrUqvOWWW84Wt7936NDhXFRUVL6bmxvx8fEnt2zZ4hMZGXnp8OHDnmPGjGm1atUqv2bNmhVdO5raQ2oAhKgJ+flw772wciXMnAnPPANKJtsUDUtFrtQdwdfXtygkJCQ/KSnJp0uXLhcBtNYqMTHx0LBhw87Yb7t+/fomZR2ntGmFAwMDi5KTk1NWr17tt2DBgsAVK1b4r1y5MqN4m8LCQqKjo6MA+vfvnzt37tyjFYk5ODi4ICMjw6P4dWZmpkd1Xv2D1AAI4XgXLhgd/VauhJdegmeflcJfiBrk7u6uP/744/3vvfde8wULFvgD9OvX7/Trr78eWDzt765duzzPnDnj0qtXr7xVq1b5FxYWcvToUbft27f79OzZ8xwYTQBpaWkeRUVFrFq1yr9nz555mZmZbkVFRYwdOzb3hRde+Hn37t1XNHG4ublRPNVvRQt/gMGDB+e+++67zW02G5s2bWrs6+tbVN0JgNQACOFIZ8/C4MGweTO8/rrR7i+EqHF+fn62Tz/9dF/v3r3DfX19i6ZNm5aTkZHh2aFDB6vWWvn7+xds2LBhf0JCQu62bdt8rFZre6WUnjVr1pHQ0NDCXbt2ER0dfW7y5MmhGRkZXj169DiTkJCQu337du8JEyZYbDabAnjuueeOVCau2bNnt5g/f/51J06ccO/YsWNUnz59Tq9YseJgfHz86fXr1zdp3bp1tLe3t+2NN97IqO7PxGHTAVcnmQ5Y1Em5ucakPt9+C0uXGj3+hahBMh1w9Vm3bp1vYmJiUFJS0j5nx1JZZU0HLDUAQjhCdrYxqc+ePUbV/913OzsiIYS4giQAQlS3o0ehb1/46SdYuxb693d2REKIXykuLi4vLi4uz9lxVCdJAISoThkZEBMDx4/DJ5/AHXc4OyIhhCiVJABCVJf0dOPK/+xZY1KfW25xdkRCCFEmSQCEqA67dkG/fsYwv5s3Q8eOzo5ICCGuScYBEOLX2r4devcGd3fYskUKfyFEnSAJgBC/xldfGW3+zZoZhX9ERPn7CCFqtRkzZlxXXceaPn16yDPPPBP0a47Rs2fPMF9f35v69OnTzn55Wlqax4033hgZGhoaPXDgwLYXL16s1AhjkgAIUVWffmr08G/Z0kgE2rRxdkRCiGowb9684Mpsb7PZKCpy3BQAjz322LGFCxf+VHL59OnTW06ZMiXr0KFDyU2aNCl85ZVXAipzXEkAhKiK1ath0CDjiv+rr+D6650dkRCiFOnp6R5t2rRpP3jw4DZt27Zt379//7Z5eXkua9eu9e3bt+8NxdutXr3ar1+/fjc8+OCD11+6dMklMjIyavDgwW0AZs6cGRQWFtY+LCys/XPPPdei+LgWiyV66NChlvDw8Pb79+/3WLVqlV9UVJQ1IiIiqnv37uHFx05NTfXu1q1bRMuWLTvMnj27UtMFA9x11115fn5+NvtlNpuNr7/+2nfcuHGnAMaPH3/i3//+d9PKHNdhnQCVUq2AZUAQoIFFWutXlFL+wArAAmQA8VrrU46KQ4hq9847MHYs3HwzfPwxNK3U35wQDVq3xd2uaie723r3yRm3z8jOu5TnErMsJqzk+ntvvDfn4VsePpGZl+l217/uusF+3fb7t6eXd86MjAyvhQsXZsTGxp4bPny45cUXXwycOXNm1tSpU0OPHj3qFhISUrhkyZLm48aNyxk5cuTppUuXtkhLS0sB2LJlS6Ply5c337lzZ6rWmi5dulhjYmLyAgICig4dOuT55ptv/hQTE5Nx9OhRtylTplg2b96cFhkZmZ+VlXV5WuB9+/Z5bdu2LT03N9fVarVGP/7449menp6/ahjerKwsN19f3yJ3d3cALBZLflZWlkc5u13BkTUAhcCjWuso4FbgIaVUFDAD2KS1DgM2ma+FqP2ys+Gpp2D0aOjVCz7/XAp/IeqA6667Lj82NvYcQEJCwolt27b5uLi4EB8ff2Lx4sX+OTk5rt99953P8OHDT5fcd/PmzT4DBgzI9fPzszVp0sQ2cODAU0lJSb4AwcHB+TExMefM7Rp369YtLzIyMh8gKCjocptAbGxsrre3tw4ODi709/cvOHLkSK24A89hQWitM4FM83meUioVuB64C+htbvY2sBl40lFxCPGrpaTA3LmwbBlcugQjRsCSJeDt7ezIhKhzrnXF7uvpa7vW+mDf4MKKXPGXVNo0vgAPPPDAiYEDB7bz8vLSgwYNOlV8NV1RjRo1spW/Fdhf7bu6ulJYWHhFQMuWLWv6/PPPhwAsWrQoo1evXufLO2ZQUFBhXl6ea0FBAe7u7mRkZHgEBQXlVyb+GukDoJSyAJ2Ab4EgMzkAOIbRRFDaPhOVUjuUUjuys7NrIkwhfqG1MZjPgAHQvj38859GtX9qKrz3nhT+QtQhmZmZHhs3bmwM8O677/r36NHjLIDFYikICgoqSExMDJ44ceLlCYvc3Nx08TTBffr0Obthw4ameXl5LmfOnHHZsGFDsz59+lw1JHDv3r3Pbd++3TctLc0DwL4JoDyjR4/OLZ4yuCKFP4CLiwu33npr3ltvvdUMYMmSJc3j4uJyK3pOqIEEQCnlA3wAPKK1PmO/ThtTEZbaDqK1XqS17qq17hoYGOjoMIUwXLpkzNzXsaMxsM9338Gf/wyHD8OCBRAZ6ewIhRCVZLFYLs6fP79F27Zt2+fm5ro99thjl68qR4wYcSI4ODi/c+fOF4uXjRo1KttqtUYNHjy4ze23335+5MiRJzp37mzt0qWLNSEhIfu22267UPIcISEhhfPmzcsYOnRou4iIiKihQ4e2ra74u3TpEpGQkND266+/9gsKCrrxgw8+8ANITEw8Mn/+/OtCQ0OjT5065TZ16tRKzbro0OmAlVLuwDrgU631HHNZOtBba52plAoGNmutr3nztEwHLBzuxAmjgP/HP+DYMYiOhunT4fe/By8vZ0cnRJXIdMBGb/24uLiwvXv37ilt/ejRo0M7dep0ftq0aXV+yuKy1Ph0wMpoZHkTSC0u/E1rgTHAX82faxwVgxDlSk832vfffhsuXDDu658+3RjTX1VqTA0hRB3Tvn17q7e3t23hwoWHnR2LMziyJ+JtQAKwWyn1g7nsjxgF//tKqQnAQSDegTEIcbXi8frnzIF168DTExIS4JFHjPZ+IUS9ERERkV/W1f+ePXtSazqe2sSRdwFsBcq6hIpx1HmFKFN+Prz/vlHwf/89BAbCs8/Cgw9Ci0qPzSGEEHVarbgXUQiHOnkSFi2C+fPh6FGwWmHxYhg1SnrzCyEaLEkARP21dy+88gq89RacP2/06n/zTYiNBRcZBVsI0bBJAiDqF62NWfnmzIG1a40pekeNMtr3b7zR2dEJIUStIZdBon4oKDAG6OnWDe64A7ZuhT/9CQ4eNEbtk8JfiAYpPT3dIywsrNb27h02bJjl+uuv7xAZGRkVGRkZtW3bNm8wJvsZO3Zsq9DQ0Ojw8PCorVu3Nqruc0sNgKjbcnON9vx58+DIEWN2vgULjF79jar970UIIard7NmzjxTP6lds5cqVTQ4cOOCVkZGRnJSU1PjBBx8M3bVrV1p1nldqAETddOAATJ0KLVvCE09AWJhxS19KCkyaJIW/EOIqKSkpHlarNerLL79sVFhYyKRJk1pGR0dbw8PDo1588cUAMK68J02a1DIsLKx9eHh41OLFi5sBrFu3zrdr164RvXv3bmexWKJHjhwZWlRURGFhIcOGDbMUbz9r1qxquaVozZo1TUeNGnXCxcWFmJiYc2fOnHE7ePBg5SYrKIfUAIi6Q2vYts1o3//oI6Mj3+9/D9OmQadOzo5OCFGe8eNbkZxcvdl5dPR5liwpdyCfH3/80XPEiBE3LFmy5Kfu3btfeOmllwKaNGlSlJycnHrhwgV18803Rw4aNOjMN99802j37t3eqampezIzM926detmjY2NPQuwe/fuxt9//31yeHh4fq9evcKWLVvWrF27dpcyMzPdi8cayMnJqfAcAMVmzZp1/QsvvBDcs2fPvH/84x9HvL29dWZmprvFYrk8uU9wcHD+wYMH3Vu3bl1Q2eOXRWoARO1XWGjcv9+9O9x+OyQlwZNPQkaGMUOfFP5CiGs4efKk25AhQ9q98847B7p3734BYOPGjX7vv/9+88jIyKhOnTpZT5065ZaSkuK1ZcsW3/j4+JNubm60atWq8JZbbjlb3P7eoUOHc1FRUflubm7Ex8ef3LJli09kZOSlw4cPe44ZM6bVqlWr/Jo1a1Z07WiuNGfOnJ8PHDiQ/OOPP6aeOnXK9emnn77OEZ9BaaQGQNRep08bt+298gocOgTt2sGrr8KYMdC4sbOjE8IhLhRcIP1EOqnZqaTmpPLEbU/g4+Hj7LCqRwWu1B3B19e3KCQkJD8pKcmnS5cuFwG01ioxMfHQsGHDrpikbv369U3KOk5p0woHBgYWJScnp6xevdpvwYIFgStWrPBfuXJlRvE2hYWFREdHRwH0798/d+7cuUftj1F8Re/t7a3Hjx9/IjExMQggODi4ICMjw6N4u8zMTI/qvPoHSQBEbZSRYXTqe+MNyMuDXr2M13Fx4Frp2jUhaqXTF0+TmpNKanYqKdkpxvOcVH469RPanCTVRbnwu6jfcWOQ3MXya7i7u+uPP/54f58+fcJ8fHxskydPPtmvX7/Tr7/+emBcXFyep6en3rVrl6fFYino1atX3uLFiwOnTJly4vjx427bt2/3mTdv3uFdu3Z57969u3FaWppHWFhY/qpVq/zvu+++7MzMTDdPT0/b2LFjc9u3b38xISHhilkA3dzcSEtLSykrtuJqfZvNxocfftjUarVeABg8eHDua6+91uL+++8/mZSU1NjX17dIEgBRf337LSQmwgcfGBPx3HOP0b7f1amTmQlRZVprjp87TmqOWcibV/WpOakczfvlQtDT1ZPw5uHcHHIzo28cjTXQijXASljzMLzcZDbK6uDn52f79NNP9/Xu3Tvc19e3aNq0aTkZGRmeHTp0sGqtlb+/f8GGDRv2JyQk5G7bts3HarW2V0rpWbNmHQkNDS3ctWsX0dHR5yZPnhyakZHh1aNHjzMJCQm527dv954wYYLFZrMpgOeee+5IZeK655572pw8edJNa62ioqLOL1u27CBAfHz86fXr1zdp3bp1tLe3t+2NN97IqO7PxKHTAVcXmQ64HisqMjr0zZljdPBr0sToxT9lCrRq5ezohKgQm7Zx+PThX67ks1NJyTEK/FMXf7m7y8fDh6jAKKwBRgFvDbQSFRhFm6ZtcHWp/totmQ64+qxbt843MTExKCkpaZ+zY6msGp8OWIhrysszBuh55RX46Sdo29ao5h87Fnx9nR2dEKUqKCpg/6n9l6/kiwv8tJw0zhecv7xdYKNArIFW4tvH/1LgB1q53vf6q9qRhXAWSQBEzTp0yJiUZ/Fio5PfbbfBSy/BXXdJ+76oNYo74pWstt97Yi8Ftl+aYVv5tcIaaGVi54mXq+2tgVYCGgU4MXrhCHFxcXlxcXF5zo6jOkkCIGrGjh1GNf/77xuvf/c7o33/llucG5do0HIv5v5SwNtd1WfkZlzREa+dfzusAVYGhw++XG0fGRBZf3rniwbJYQmAUmoJEAcc11pHm8v8gRWABcgA4rXWp8o6hqjjiorg3/82Cv4tW4yq/alT4eGHoXVrZ0cnGgitNVnnsq6qtk/NTiXzbObl7TxdPYkIiKDb9d0Y03GMUXUfaCXMPwxPN08nvgMhHMORNQBLgX8Ay+yWzQA2aa3/qpSaYb5+0oExCGc4d86YgnfuXNi/3yjs58yBCRPAz8/Z0Yl6yqZtHDp96Krb6kp2xPP18MUaaCX2htgr2ucd1RFPiNrKYQmA1vorpZSlxOK7gN7m87eBzUgCUH/8/DP84x+wcCGcOmVU77/wAgwdCm7S2iSqR0FRAftO7ruq2j79RHqpHfHuaX/P5fb5qMAoQnxDpCOeENR8H4AgrXVxndsxIKisDZVSE4GJAKGhoTUQmqiy776Dl1+Gf/0LbDa4+26YPt0YuleIKjpfcJ70nPSrbqvbe3IvhbbCy9uFNgnFGmDljtZ3XG6ftwZYad6ouROjF3XZjBkzrvvrX/96rDqONX369BAfH5+i5557Lquqx3B1de0SFhZ2ASAkJCT/iy++2AeQlpbmER8f3zY3N9etQ4cO5z/44IOfvLy8Knxvv9Muy7TWWilVZqBa60XAIjDGAaixwETF2Gywfr1Rtb95M/j4wEMPGW38bdo4OzpRhxR3xCtZbW/fEc9VuXKD/w1YA6wMiRxyudpeOuIJR5g3b15wZRIAm82G1hpXB93J5OnpaSttNMHp06e3nDJlStbEiRNPjRw5MvSVV14JePLJJ7MretyaTgCylFLBWutMpVQwcLyGzy9+rfPn4e23jfb9//3PmI73xRfhvvugaVNnRydqqeKOeCVvq0vJTuHY2V/+zxZ3xLul5S2MvWns5YJeOuKJqkpPT/fo379/WIcOHc4nJyc3Cg8Pv7By5cqMpKSkxvPmzWuxcePG/QCrV6/2e+211wLDwsIuXrp0ySUyMjIqPDz8wtq1a3+aOXNm0LvvvhsAkJCQkP3MM88cT09P9/jNb34T3qlTp7O7d+9uvGHDhr27du3yeuaZZ64vKipS/v7+hV9//fX/AFJTU727desWcfToUY/JkydnPfXUU7+67LPZbHz99de+a9asOQAwfvz4EzNnzgypzQnAWmAM8Ffz55oaPr+oqsxMYyKe11+HkyeN4Xnfew+GDQP3ap2iWtRhNm3jYO7BUse4z72Ye3k7Xw9fogKj6N+u/+W2eWuAFUtTi3TEq++6dYu4atndd59kxoxs8vJciIkJu2r9vffm8PDDJ8jMdOOuu264Yt327enlnTIjI8Nr4cKFGbGxseeGDx9uefHFFwNnzpyZNXXq1NCjR4+6hYSEFC5ZsqT5uHHjckaOHHl66dKlLYqvuLds2dJo+fLlzXfu3JmqtaZLly7WmJiYvICAgKJDhw55vvnmmz/FxMRkHD161G3KlCmWzZs3p0VGRuZnZWVd/iLv27fPa9u2bem5ubmuVqs1+vHHH8/29PSscM12fn6+S3R0tNXV1VU/9thjxxISEnKzsrLcfH19i9zN/78WiyU/KyvLo5xDXcGRtwG+h9HhL0ApdQR4FqPgf18pNQE4CMQ76vyimvz4o9G+v3y5MS3vkCFG+/5ttxnj9YsGyb4jnv1tdWk5aVwovHB5uxaNW2ANsDKi/YjLt9VZA6zSEU/UqOuuuy4/Njb2HEBCQsKJefPmtXBxccmKj48/sXjxYv+HHnroxHfffefz4Ycf/lRy382bN/sMGDAg18/PzwYwcODAU0lJSb7Dhw/PDQ4Ozo+JiTlnbte4W7dueZGRkfkAQUFBl6cFjo2NzfX29tbe3t6F/v7+BUeOHHG74YYbKjyxz969e3e1adOmICUlxaNfv34RnTt3vuDv71+paYdL48i7AH5fxqoYR51TVBObDT75xGjf37QJGjUyxuefOtWYklc0GOcLzpOWk3ZVtf2+k/tK7YjX29L7crW9dMQTpbrWFbuvr+2a64ODCytyxV9SadP4AjzwwAMnBg4c2M7Ly0sPGjTolHslazMbNWpkq8h29lf7rq6uFBYWXhHQsmXLmj7//PMhAIsWLcro1avXefv1bdq0KQCIiorKv/XWW/O2b9/eaMyYMafy8vJcCwoKcHd3JyMjwyMoKCi/MvHLvVniFxcuwDvvGFf8qakQEgJ//StMnAjNmjk7OuFAcfAFIAAAExFJREFUpy6cuuq2utScVA7mHryqI15UYBRDI4derrqPCIiQjniiVsvMzPTYuHFj4759+5579913/Xv06HEWwGKxFAQFBRUkJiYGf/LJJ/8r3t7NzU1funRJeXp66j59+pwdP3685c9//vMxrTUbNmxotnTp0gMlz9G7d+9z06dPb52WluZR3ARgXwtwLaNHj84dPXp0bmnrsrOzXX18fGze3t46MzPTbceOHT5//OMfj7m4uHDrrbfmvfXWW80mTpx4asmSJc3j4uJKPUZZJAEQkJUFr71mPHJyoFMnIxEYPhw8KtWkJGoxrTXHzh4rdWpa+454Xm5eRDSP4NaWtzLupnGX2+fb+beTjniiTrJYLBfnz5/fYuLEiY3CwsIuPvbYY5c7yo0YMeLEq6++6ta5c+eLxctGjRqVbbVao6Kjo8+vXbv2p5EjR57o3LmzFYxOgLfddtuF9PT0K/45hoSEFM6bNy9j6NCh7Ww2G82bNy/Ytm3b3l8b+w8//OD10EMPtVZKobXmkUceOdalS5eLAImJiUfuueeeG2bPnn19+/btz0+dOrVSsy7KdMANWXKycbX/zjuQnw+DBhnt+3fcIe37dVhxR7zSpqY9fen05e38PP2u6IBXXG0vHfHqD5kO2LgLIC4uLmzv3r17Sls/evTo0E6dOp2fNm1anZ+yuCwyHbAwaA2ffWa073/2GXh7G0P0PvIIhIc7OzpRCflF+UZHvBLV9uk56Vd1xIsKjGJkh5FXzEEf7BMsHfFEg9a+fXurt7e3beHChYedHYszSALQUFy8aPTknzMH9uyB666Dv/zF6NzXXDpq1Wbn8s+RfiL9qtvqSnbEa92kNdZAK30sfa64qvf39ndi9EI4V0RERH5ZV/979uxJrel4ahNJAOq77Gzj3v1XX4Xjx+HGG2HpUhgxAjylPbc2OXnhZKlT0x48ffDyNq7K1ZiaNtDK3ZF3X662l454ohaz2Ww25eLiUvvbm+shm82mgFLvVpAEoL5KTTXa95ctg0uXYMAAo33/zjulfd+JtNZkns28oqAvbp/POvfLUOHFHfF6tOrBhE4TLlfbt/Nvh4erdMwUdUpydnZ2VGBg4GlJAmqWzWZT2dnZTYDk0tZLAlCfaA1ffAGJifDxx+DlBWPGGO37Vquzo2tQbNpGRm5GqWPcl+yIFxUYxYCwAb90yAu00rpJa+mIJ+qFwsLC+44dO/bGsWPHouH/27v34Liq+4Dj359WK2lXEtZjZVl+YRscG5sYsD3BJC0EAzaYDLQh6RBoQ0iYhKY0TUMJkMdMwqSBph2mZEiaEnBaOoQQKE0hA2VcHkOHwWBs/FCwARdS4zcSxpa0K+1q9esf56x0Je1aD8u70u7vM3Nn7z337H3sWel37rln76Gs0MdTYvqA1t7e3huyrbQKQDHo6XEj8d19N2zfDtOnwx13wI03QlNToY+uqGU64g39Wd2utl109/b/qojm6mbOaDqDaz56zaD789YRr7iouuEyOjqgs9NNwfmhy6PJt23b1H7+1ooVKw4DVxT6OMxwVgGYyg4cgPXr4d574eBBWLoUHngArrnGXf2bE5JMJ2mPt9MWb6M94V/j7ew5umfQE/HSOvCsj0xHvNXzVvffn7eOeJOTqusbO9pAPJp8XV1uu6MRCkFtrZtqatxUW+v65GbmM+nGnAxWATgBPd0pNj79Ihf88Ul+urEq7N0LW7YMnvbvd+vXrnUj9F1yid3fz6G7t5v2ePugQN4Wbxsc3Ies60h2ZN1WSEIsbFzIkqYlXHXGVf335xc1LqK6ojrPZ1Y6enrGduU8Ur7OTkiP8mnqZWXDg3JNDcyaNTh4Z5vPta6iwv5cTWFZBWCc/mv9oyz9+tUsTivxQ11EayITs+G+PnjnneHBvr3drS8rg8WLXWe+5cthzRp35V9CEqlEf7AOBuyhgTy4rivVlXN7tRW1xKIxGqONxKIxFscW0xhx8/2v0YHlpuom64g3glRqdEF5LAE7NeqhU7IH3unT4bTTxhewq6osWJviYxWAUdrx8g42/uC7lDU286UH/5mPXX4RG+9s4cjFn+Wq8f5j6O2FN990Af711wdejx1z68NhOPNMNwLf8uVuWrbMDc5TJOKpeO5AHm+nLTF8XTwVz7m9aZXT+oP1jJoZLG1amjWQZ9Iao40lH8zT6eFXxmO9Tz10vqdn9PuPRocH3vp6mDt35KvobMvRqKsnG2OOzyoAx/H+/jae/M73mPPcI1y4p42PKjyxaC4ADc0NrHt77+g3lky6B/AEr+q3bXMD8IB7It9ZZ8G11w4E+6VLp8xv9VWVrlRX1kAeDOZD1wU7yg1VV1XXH6xn1s5kWfOynIE8Fo3REGkgHBrbaF5TTV/f8E5mJxqwE4mR95tRVTU88NbWQkvL6Jq9h85XV7t74caY/LMKwBB96T7KQu7y4dULzuKLu/ezpzbEwx8/j3lf+xZX/MmnRt5IPO564weDfWvrQBtmba0bcOcrXxkI9osWQfnkKA5VpSPZkTuQ52huT6azj0QpCPWR+v5gPXfaXM5pOYdYJDaseT2z3BBpoLxscnwe46XqgutE9AQPdjIbrXA4e1BuahrfPevqardNY0xxmNr/YSfQcw89wXs/vZMLWjdx5D9e4pzV55L+i+/zeGcHV9z6l/xZOMtHperuzWeu7DPN+Dt3uks1cF16ly93D+HJBPsFC/LWRqmqHOs5NrzDW7bgHliX6st+w7VMymiINPQH63l181jZsjJnII9FY9RX1RfsN+3ptGt8Od6USo2cJzhlgvpIAbuzc+BrMJKysuE9wmtqYM6cE+tkZowxuRSkAiAilwL3ACHgflW9K5/7z1zlt768lW1fu54z9rzF6sNxegVeOLWJ0MFD0NnJFevOh3374NePuB73+/a51+B8MnDVO3OmC/Cf/vRAsJ8zZ1y9h/q0j+7ebhKpBIneBIlUwi37+USvWz7affS4gbw90T7oefFBIQnREGnob0o/veF0Vs1alTOQx6Ix6qrq0L6y4wfROCQ/hMNJ2DuGwDpRATo4jTYAj4VI9sA7Y4b7vfZYmsAzy5WV1snMGJNfeR8OWERCwFvAJcBeYBPwOVV9I9d7xjsccLI7yYaf/5Ij//Ms0be3M+vQHuYdPcb2hcu45Ju3cGDHThp/9APaImHi9TNoaW4h2nUU9h9AMh3xAvqiEVIt00lOj9HT3EiiuZHuWB1HT53BwY/M4oNpVXQlEySS3cRTCeLJBPGUD9ipwcG7p7ebRDpBd6+betLd9KQTdKcT9KQTpDR7c3ouIcqpLmskSowojUQ0RmVfI5XpGBW9jVT0xihPxQj1NBLqiSGJRrR7Gqnk8GBeiKAK7g5IRcXET+HwxG/PgrWZCibDcMBm8ipEBeA84HuqutYv3w6gqnfmes94KwDPnzaXC98ZeZTHZBnsrx2Y9p0SmA+kd4zn2TrpMKQi0BuB3qqB+ZRf7p8fsv64eQPzPadAPOZeGR6VJjqoTnQwtaBqzMljFQBzPIW4BTALCEblvcC5QzOJyJeBLwPMnTt3XDvacta5HAr3cKimlv3TmuiMROisrKSzMkI8XEVXZYTOygjJ8ipCUkFIwoQID7xSQbmEmSNh5qXDhOJhyiVMSFy6mw9TUVZJRShCpUSoCFVRFYpQFYpQUVZFuDxEKOR6OpeV0T+fK22k5aFp2QJyJi0ctp9DGWOMyW7SdgJU1fuA+8C1AIxnGzc//uiEHpMxxhhTLApxfbgPmBNYnu3TjDHGGJMnhagAbAIWish8EakArgaeKMBxGGOMMSUr77cAVLVXRG4CnsH9DHC9qv4u38dhjDHGlLKC9AFQ1aeApwqxb2OMMcYU5haAMcYYYwrMKgDGGGNMCbIKgDHGGFOCrAJgjDHGlKC8Pwp4PETkfeD/RsgWA9rycDiTjZ13abHzLi0net6nqmrTRB2MKS5TogIwGiLyWik+89rOu7TYeZeWUj1vkx92C8AYY4wpQVYBMMYYY0pQMVUA7iv0ARSInXdpsfMuLaV63iYPiqYPgDHGGGNGr5haAIwxxhgzSlYBMMYYY0rQlK8AiMilIvKmiOwWkdsKfTzjISJzROR5EXlDRH4nIn/l0xtEZIOIvO1f6326iMiP/TlvF5HlgW1d5/O/LSLXBdJXiMgO/54fi4jk/0yzE5GQiLwuIr/1y/NF5BV/rI/4YaMRkUq/vNuvnxfYxu0+/U0RWRtIn5TfDxGpE5HHRGSXiOwUkfNKobxF5K/9d7xVRB4WkapiLW8RWS8ih0WkNZB20ss41z6MGUZVp+yEG074f4EFQAWwDVhS6OMax3m0AMv9fC3wFrAE+BFwm0+/Dfg7P78OeBoQYBXwik9vAN7xr/V+vt6ve9XnFf/eywp93oHz/wbwS+C3fvnXwNV+/mfAn/v5rwI/8/NXA4/4+SW+7CuB+f47EZrM3w/gX4Eb/HwFUFfs5Q3MAt4FIoFy/kKxljdwPrAcaA2knfQyzrUPm2waOk31FoCPAbtV9R1VTQK/Aq4s8DGNmaoeUNUtfr4D2In7Z3klLlDgX//Iz18JPKjORqBORFqAtcAGVf1AVY8AG4BL/bpTVHWjqirwYGBbBSUis4HLgfv9sgCrgcd8lqHnnfk8HgMu8vmvBH6lqj2q+i6wG/fdmJTfDxGZhgsODwCoalJVP6QEyhs3BHlERMqBKHCAIi1vVX0R+GBIcj7KONc+jBlkqlcAZgHvBZb3+rQpyzdzngO8AjSr6gG/6iDQ7Odznffx0vdmSZ8M/hH4JtDnlxuBD1W11y8Hj7X//Pz6oz7/WD+PQpsPvA/8wt/6uF9Eqiny8lbVfcA/AHtwgf8osJniL++gfJRxrn0YM8hUrwAUFRGpAf4d+LqqHguu87X8ovrNpoh8CjisqpsLfSx5Vo5rGv4nVT0H6MI11fYr0vKux12dzgdmAtXApQU9qALKRxkX4/fITJypXgHYB8wJLM/2aVOOiIRxwf8hVX3cJx/yTX3418M+Pdd5Hy99dpb0QvsEcIWI/B7XXLsauAfX/Fnu8wSPtf/8/PppQDtj/zwKbS+wV1Vf8cuP4SoExV7eFwPvqur7qpoCHsd9B4q9vIPyUca59mHMIFO9ArAJWOh7EVfgOgo9UeBjGjN/X/MBYKeq3h1Y9QSQ6fV7HfCfgfTP+57Dq4CjvsnvGWCNiNT7q601wDN+3TERWeX39fnAtgpGVW9X1dmqOg9Xds+p6rXA88BnfLah5535PD7j86tPv9r3Gp8PLMR1kJqU3w9VPQi8JyKLfNJFwBsUeXnjmv5XiUjUH1fmvIu6vIfIRxnn2ocxgxW6F+KJTrjes2/hev9+u9DHM85z+ANcM912YKuf1uHudz4LvA38N9Dg8wvwE3/OO4CVgW19EdcpajdwfSB9JdDq33Mv/imQk2UCPsnArwAW4P6h7wYeBSp9epVf3u3XLwi8/9v+3N4k0ON9sn4/gLOB13yZ/wbXw7voyxv4PrDLH9u/4XryF2V5Aw/j+jqkcK0+X8pHGefah002DZ3sUcDGGGNMCZrqtwCMMcYYMw5WATDGGGNKkFUAjDHGmBJkFQBjjDGmBFkFwBhjjClBVgEwRU1E0iKyVUS2icgWEfn4CPnrROSro9juCyKycpzH9JSI1I3nvcYYM1GsAmCKXUJVz1bVs4DbgTtHyF+HG4XupFHVdeoG/zHGmIKxCoApJacAR8CNuyAiz/pWgR0ikhk17i7gNN9q8Pc+760+zzYRuSuwvc+KyKsi8paI/OHQnYlIi4i86LfVmskjIr8XkZiI3OjXbRWRd0Xkeb9+jYi87I/tUT9GhDHGTCh7EJApaiKSxj1ZrQpoAVar6ubMcLSqekxEYsBG3CNlT8U9kfBM//7LgO8CF6tqXEQaVPUDEXkB2KyqN4vIOuAbqnrxkH3fDFSp6t+KSMjvr8OPfbBSVdt8vjDwHG4c95dxz8i/TFW7RORW3JPx7jiZn5MxpvSUj5zFmCktoapnA4jIecCDInIm7tGrPxSR83FDEc8i+7CpFwO/UNU4gKoGx3fPDNq0GZiX5b2bgPU+wP9GVbfmOMZ7cM+5f9KPkLgEeMk94p0KXKXAGGMmlFUATMlQ1Zf91X4T7pnxTcAKVU35q/KqMW6yx7+myfK3pKov+grG5cC/iMjdqvpgMI+IfAHX6nBTJgnYoKqfG+OxGGPMmFgfAFMyRGQxEMINKTsNOOyD/4W4IAzQAdQG3rYBuF5Eon4bDWPY36nAIVX9OXA/bsjf4PoVwN8Af6qqfT55I/AJETnd56kWkY+M7UyNMWZk1gJgil1ERDJN7wJcp6ppEXkIeFJEduBG5dsFoKrtIvKSiLQCT6vqLSJyNvCaiCSBp4BvjXLfnwRuEZEU0IkbsjXoJqABeN4397+mqjf4VoGHRaTS5/sOboQ7Y4yZMNYJ0BhjjClBdgvAGGOMKUFWATDGGGNKkFUAjDHGmBJkFQBjjDGmBFkFwBhjjClBVgEwxhhjSpBVAIwxxpgS9P/wYDLfMIL/6wAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "n_features = [2, 10, 50]\n", + "max_batch_size = 100000\n", + "\n", + "plot_absolute_time(results, n_features, max_batch_size=max_batch_size)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "ec9d0fbb", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiIAAAEWCAYAAABbt/wMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdd3hUZfbA8e9JQocgnSQQQoeAdFEhriCKLmJhFVZBVCygKCpY1sIiouvKz7WBqIggoigqi4qKK6KwVGEDCNKVIiV0aaGknt8f700cQsoEMkxCzud55knmzi1n7tyZOfNWUVWMMcYYY4IhJNgBGGOMMab4skTEGGOMMUFjiYgxxhhjgsYSEWOMMcYEjSUixhhjjAkaS0SMMcYYEzRFNhERERWRBrk8/paI/N3n/r0isltEEkWkymkcL8Y7ZtjpxlxQvOdQL9hxnC0iMlFEnsvH+rleG4WNiHwjIrcFOw5/iEi0d/2Fnsa2ZUTkSxE5JCKfBiI+c3pEpI+IzAx2HKZ4yjMREZEtIpIsIlWzLF/ufeDHePcnevevy7LeK97y2737t4tImvdhligim0XkXRFpVGDPClDVe1T1We+YJYCXga6qWl5V9wfyy8p7jvMLaF9zROQu32Xec9hUEPsvCIUhSRORi0VkYbCO7y8RGS4iH/guU9U/q+p7wYopN977//KM+6q61bv+0k5jdzcCNYAqqtqzAGIrKSJTvRhVRDpleVxEZKSI7PduI0VEfB5vJSJLReSY97fVmcYUDPn9vMnu/aqqk1W1a2AiNCZ3/paIbAZuzrgjIucDZbNZbwNwq896YUAvYGOW9RapanmgInA5cBxYKiLN/Q89Z9n8WqsBlAZWF8T+TcEqoATmamBGAeyn2AhC4lgH2KCqqfndMJdY5wO3ALuyeaw/cD3QEmgBXAMM8PZXEvgC+ACoBLwHfOEtDxgvOSqyJdHGBISq5noDtgBDgf/5LPsX8BSgQIy3bKK3fDdQyVvWHfgG92Fxu7fsdmB+Nsf5CpiaSxyPAjuBBOAO79gNfI79Ju6L6CguuZkIPAc08pYpkAj8AMz17h/1lv01m+OFes9nH7AJuM/bJsx7vCIw3otph3esUKApcAJI8/Z90Fu/lLe/rd45egso43O864CfgMO4xO0q4B/efk54+3rdW9f3uVcEJgF7gd+81yrE91x7xz2ASyj/nMdr/QSwxlv/XaC099gq4BqfdUt456a195wyzm8icDEuyR3qxbTHi7Git22Mt/6d3rZzveVxwELgILCNP66ZicAY4GvgCLAYqJ8l9mVAm2zOT5y3r07e/SbAd8DvwHqgl88+8jqXC4DXgUPAOqCLz7a3466TI9557pPN+b0KSAZSvPO0wls+B7gry3Fe8c7DJqCDt3ybdy5v89lnrtdVluP77ns/7pqtj3tP7Pdez8nAed767wPpuB8KicBjPq9dxvsgEpjunc9fgbtzOPYzWZ77nZzGNZLLtbs94zX2WbYQ6O9z/07gR+//rrj3rfg8vhW4Kof9zwH+CSzBvUe/ACr7PH4Rf1y7K3xj8bb9h3fujwMNvGXPedskAl8CVbzzfxj4H398tp50zn2vGXL+vLkaWO7taxswPMvzzPp+vR2fz2XcNfc/3LX+P6BDlmM/6z2fI8BMoKr3WGlccrffOxf/A2rk9T1jt+J9y3sF9+V0Oe5Duynuy3Y77tdN1kTkOeBt4F5v2Se4khR/EpE7gN05xHAV7kO2OVAO+JBTE5FDQEfch1vpjHi8x7N7I2dun8Mx78F92dQGKgOzOfkD+DNgrBdPddwH1ICcniPuw3+6t68KuA+ef3qPtffiv8KLPwpo4j02B+9LKrvYcR/eX3j7jMGVSt3pE0cKcLf3ut2LS+Qkh+e8BZdwZDznBT7n8DHgY591rwN+zuX83oH7YqoHlAemAe9nWX+Sd/7K4K6nI971UgL3odzK5/Xd752nMNyH9RSfY0Xg86WScX5w1802oL23vJx3v5+3n9a4L99YP89lKjDYi++v3mtW2dvvYaCxTzzNcjjHw4EPsizLfI19jtPPe82ew31xjMElHV2981Q+r+sqm2Nn7HuQ9/zLeOfpCm/f1XBJ+qtZ3/8+9096rb3138C951rhkrjL/Hnu5PMayeNzKrtE5BBwoc/9dsAR7//BwDdZ1v8KeDiH/c/BXWMZn0H/znguuPfrfqAb7v17hXe/ms+2W4Fm3nkv4S37FZcIVsQl/xtwn7Vh3vN+N5f3V9ZrJuvnTSfgfC+eFrjPz+tz2V/mPnDX0gGgrxfLzd79Kj7H3oj7kVfGu/+C99gA3DVYFnf9tgXC8/qesVvxvuWniPB9XLXLFcBa3JsyO5OAW0XkPOBS4HM/95+AewNkpxfuTblKVY/iPtCy+kJVF6hquqqe8POYuemF+0Depqq/434NASAiNXAfOg+p6lFV3YP7Qrgpux159dL9gcGq+ruqHgGe91n/TmCCqn7nxb9DVdflFaBXBXUT8ISqHlHVLcBLuA+QDL+p6jh1dfrv4b4ka+Sy29d9nvM/+KNK7gOgm4iEe/f74q6JnPQBXlbVTaqaiCtpuSlLEftw7/wdB3oDs1T1I1VNUdX9qvqTz7qfqeoSdcX6k3Ffehm6Af9RVfVZ1hOXKP5ZVZd4y7oDW1T1XVVNVdXluC+Unn6eyz24ayJFVT/GJedXe4+lA81FpIyq7lTVM6kG3OzFmAZ8jEsMR6hqkqrOxJUsNPDjuspOgqqO9p7/cVX91bvuklR1L64t1aX+BCkitXHJ/99U9YT3er2DT/VsHvJ7jeRXeVwykuEQUN47b1kfy3i8Qi77e9/nM+jvQC/vurkFmKGqM7z373dAPO66zDBRVVd75z3FW/auqm5U1UO4kuONqjrLu8Y/xSXKp0VV56jqz148K4GP8PN1xV3Tv6jq+168H+F+lF3js867qrrBe10+4Y/3YwruR0QDVU1T1aWqevh0n4cpHvJTR/w+7tdPXVyykS1VnS8i1XBVN1+p6nGf9mG5icIV72YnEljqc/+3bNbZ5s9B8iEyyz59j1kH96tmp89zC8klhmq4XwhLfdvK4X4xgPuiOZ32DVW9OHxj+w13LjNk1p2r6jHv+OVz2WfW5xzpbZsgIguAG0TkM+DPwIO57Ccym7jCODkJ8j1WbU5tS+TLtw3AMU5+Dt1wpWS+HgImqeoqn2V1gAtF5KDPsjDcte3PudyRJdn5DYhU1aMi8lfgEWC8d54e9ieZzMFun/+PA6hq1mXlyfu6ys5J16iXVL8GXIL7Eg7B/fr1RySQkQBl+A1X8uDv9vm5RvIrEQj3uR8OJKqqikjWxzIeP0LOsr43SuCumzq4ZNb3i7oErhQ1u20zZH1Ns3uNT4uIXAi8gCvBKYkr8fK3p1LW1wVy+Vzh5Pfj+7j38hTvx+gHwFM+yZcxp/C7RERVf8PVfXfDFaHm5gPgYXJJWLLRA5iXw2M7cRd3hujsQszHsfyR2zG3AUm4etHzvFu4qjbLIZZ9uA+WZj7rV1TXYDdjf/VziCO357UP9wukTpY4cyqt8kfW55zgc/893K+/nrgGxxnHyS7GhGziSuXkD1vf7XI7BznyekRdimv34asncL2I+CZL24D/+rwG56nrAXIv/p3LKDk5q848P6r6rapegStxWgeMyyHkgrxO87qu/Dn+896y81U1HPf6Si7r+0oAKouIbylCfq6//F4j+bUa11A1Q0v+aLC+GmiR5fVsQe4N2rO+N1Jwr8E2XGmJ73VVTlVf8Fn/TJ7HUe+vbweBmnns+0NclV1tVa2IazskuazvK+vrAn6+rl5p4TOqGotrZ9Id/0vITDGV39bbd+Lqf4/msd4oXBXO3NxWEpFQEakrIqNxdZrP5LDqJ8DtIhIrImWBp/MXdrZ24+qmc/IJ8ICI1BKRSsDjGQ+o6k5cA62XRCRcREJEpL6IZBR97gZqZbTAV9V03BfTKyJSHUBEokTkSm/98UA/Eeni7StKRJrkFadXdP8J8A8RqSAidYAhuETwdN3nPefKuFKtj30e+xxogysJ8U0y9+KqJnzj/AgY7L2+5XFfeB9rzj0mJgOXi0gvEQkTkSp+dqeMA1ZmU/ybAHQBHhSRe71lXwGNRKSviJTwbheISFM/z2V13DVRQkR64tpMzRCRGiJynYiUwyWoid75yM5uIKYgek74cV35owIu3kMiEoVrFJ413pyuv224xpb/FJHSItIC9xnh7/WX32vkFCJSSkRKe3dLenFkfOFOAoZ45yQS9+NoovfYHFwDzwe8fdzvLf8hl8Pd4vMZNALXuD4N93yvEZErvc+00iLSSURq+fs8cuNVme3wjh8qIndwctJ+0ueNpwKutOqEiLTHVX1myO796msG7n3S23sv/hWIxb1/ciUinUXkfK/K6jAuWcvpvWAMkM9ExKvPjPdjvd9V9fssxdi+LvaKRg/jPhDCgQtU9ecc9vcN8CruQ+JXcv+w8Ndw4D0ROSgivbJ5fBzwLa4F/DJOLQW6FVfkmdHDZCru1zBefKuBXSKyz1v2Ny/2H0XkMDALaOw9vyW4xomv4Oqp/8sfv0heA24UkQMiMiqbOAfhfjFtwjUK/hCY4N8pyNaHuCRrE66qJHMgMa8++N+46rlpPsuP4fUK8M7nRV4MGdV5m3Et+wfldFBV3YorbXsYV0X3Eyf/ms1Jjt12vX12AR4Xkbu8KoSuuDYUCbji5ZG4YmvI+1wuBhrifgX/A7hRVffj3kdDvH3+jiuhuZfsZRSP7xeRZX48v7zkeF356RlccnkI1ysp63X+T2Co97o+ks32N+MaPybgGnA/raqz/Dx2vq6RHKzHlQpF4d6vx/njvTMW13DyZ1wj7K+9ZahqMq5r76243h134BpzJudyrPdxicwuXOPcB7x9bcM13n4S9yW/DZfQFWQ33bu9fe7HNXr1HTMnu8+bgcAIETkCDMMl2XjxZvd+xefx/biSjIe94z0GdFfVfeStJu6z8DCuLeF/yb0tmTGZvQyMQUS24Fri5/hFIiLDgEaqestZCywXIrIGlxCsCfBxbsedm7hAHscUTiIyB9dL5p1gx2LMuSbow5WbosOrrrmTk3uSBI1XFD0p0EmIMcaYwLER/oxfRORuXJHzN6qaa9ufs0VVk7M0CDTGGFPEWNWMMcYYY4LGSkSMMcYYEzRFoo1I1apVNSYmJthhGGNMkbJ06dJ9qlot2HEYk5sikYjExMQQH59nr2FjjDE+RCS7UaiNKVSsasYYY4wxQWOJiDHGGGOCxhIRY4wxxgSNJSLGGGOMCRpLRIwxxhgTNJaIGGOMMSZoLBExxhhjTNBYImKMMYXQiRMwYQJMmRLsSIwJLEtEjDGmENm7F559FurUgTvvtETEnPssETHGmEJg7VoYMACio2HYMGjXDmbNgs8+C3ZkxgRWkRji3RhjzkWq8MMP8PLLMGMGlC4Nt94KDz0ETZsGOzpjzg5LRIwx5ixLTnZVLi+/DCtWQPXq8MwzcO+9UM2mqDPFjCUixhhzluzfD2PHwuuvw86d0KwZjB8PvXu70hBjiqOAJyIiEgrEAztUtbuI1AWmAFWApUBfVU0OdBzGGBMsGzbAq6/CxIlw/Dh07Qrvvuv+igQ7OmOC62w0Vn0QWOtzfyTwiqo2AA4Ad56FGIwx5qxShf/+F667Dpo0cSUfN98MP/8M334LV15pSYgxEOBERERqAVcD73j3BbgMmOqt8h5wfSBjMMaYsyklBSZPdr1eOnWChQvh73+HrVtdMtK8ebAjNKZwCXTVzKvAY0AF734V4KCqpnr3twNR2W0oIv2B/gDR0dEBDtMYY87MgQMwbhyMGgU7dkDjxq49SN++UKZMsKMzpvAKWImIiHQH9qjq0tPZXlXfVtV2qtqumjUjN8YUUhs3wgMPQO3a8Le/uQTkq69gzRro39+SEGPyEsgSkY7AtSLSDSgNhAOvAeeJSJhXKlIL2BHAGIwxpsCpuiqXl16Czz+HsDDX/mPwYGjVKtjRGVO0BKxERFWfUNVaqhoD3AT8oKp9gNnAjd5qtwFfBCoGY4wpSKmp8PHHcNFFEBcHc+bAE0/Ali3w3nuWhBhzOoIxjsjfgCki8hywHBgfhBiMMcZvhw7BO++49h9bt0KDBjBmDNx2G5QrF+zojCnazkoioqpzgDne/5uA9mfjuMYYcya2bHHJxzvvwJEjcOmlMHo0dO8OITZTlzEFwkZWNcaYLH780Q2//u9/u4SjVy8YMgTatg12ZMaceywRMcYYIC3NzXT78suwaBFUrAiPPAL33+96xBhjAsMSEWNMsXbkCEyYAK+9Bps3Q716rjqmXz8oXz7Y0Rlz7rNExBhTLG3b5hKOt9+Gw4ehY0f417/ckOyhocGOzpjiwxIRY0yxEh/vql8++cTdv/FGN/7HhRcGNy5jiitLRIwx57y0NPjyS5eAzJsHFSrAgw+6EVHr1Al2dMYUb5aIGGPOWUePwsSJ8Oqr8OuvLul4+WW4804IDw92dMYYsETEGHMO2rEDXn/dTTp34ICrdnn+eejRww3HbowpPOwtaYw5ZyxfDq+8Ah99BOnpLvEYMgQ6dAh2ZMaYnFgiYowp0tLTYcYMV+Uye7brcnvffa79R716wY7OGJMXS0SMMUXSsWMwaZIrAdmwAWrVghdfhLvugvPOC3Z0xhh/WSJijClSdu1yE869+Sbs3++GXf/wQ9cNt0SJYEdnjMkvS0SMMUXCypWu9OPDDyElxQ08NmQIxMWBSLCjM8acLktEjDGFlip8+y289BLMmgVly8Ldd8NDD0GDBsGOzhhTECwRMcYUOidOwAcfuBKQNWsgMhL++U/o3x8qVw52dMaYgmSJiDGm0NizB954w9327oVWreD996FXLyhZMtjRGWMCIWCJiIiUBuYCpbzjTFXVp0VkInApcMhb9XZV/SlQcRhjCr81a1zpx/vvQ1ISdO/u2n906mTtP4w51wWyRCQJuExVE0WkBDBfRL7xHntUVacG8NjGmEJO1bX7ePll+M9/oHRp6NfPtf9o3DjY0RljzpaAJSKqqkCid7eEd9NAHc8YUzQkJbmRT19+GX7+GWrUgGefhXvugapVgx2dMeZsCwnkzkUkVER+AvYA36nqYu+hf4jIShF5RURK5bBtfxGJF5H4vXv3BjJMY8xZsG8fPPecm3iuXz+37N134bffYOhQS0KMKa4CmoioapqqtgJqAe1FpDnwBNAEuACoDPwth23fVtV2qtquWrVqgQzTGBNA69a50o7ateHvf4c2beC772DFCrj9diiV7U8RY0xxEdBEJIOqHgRmA1ep6k51koB3gfZnIwZjzNmj6uZ9ueYaaNoUJk6EW26BVavcvDCXX26NUI0xTsASERGpJiLnef+XAa4A1olIhLdMgOuBVYGKwRhzdiUnu54vbdrAZZfB4sUwfDhs3QrjxkGzZsGO0BhT2ASy10wE8J6IhOISnk9U9SsR+UFEqgEC/ATcE8AYjDFnwe+/w9ixMHo07NwJsbHwzjvQp4/rDWOMMTkJZK+ZlUDrbJZfFqhjGmPOrl9+gddec41Ojx2DK66ACRPgyiut6sUY4x8bWdUYky+qMG+e6347fTqEhbmSjyFD4Pzzgx2dMaaosUTEGOOXlBSYOtUlIPHxUKUKPPUUDBwIERHBjs4YU1RZImKMydXBg66h6ahRsH07NGoEb74Jt97qZsM1xpgzYYmIMSZbmza59h/jx8PRo9C5s0tAunWDkLPS8d8YUxxYImKMyaQKixa56pfPPnMJx803w+DB0PqUpufGGHPmLBExxpCaCtOmuQRk8WI47zx47DG4/36Iigp2dMaYc5klIsYUY4cPu6qX115zc77Urw+vvw633Qblywc7OmNMcWCJiDHF0G+/ucan48bBkSNwySUuGeneHUJDgx2dMaY4sUTEmGJk8WJX/fLvf7v7vXq59h8XXBDcuIwxxZclIsac49LS4IsvXAKyYAFUrOgGHxs0yM2Ia4wxwZTvREREQoDyqno4APEYYwrIkSNu6PVXX4XNmyEmxv1/xx1QoUKwozPGGMev0QBE5EMRCReRcrjZcteIyKOBDc0Yczq2b3c9XmrXhgcfdKOeTp0Kv/7q7lsSYowpTPwdlijWKwG5HvgGqAv0DVhUxph8W7rUzflSty689BJ07erGBFmwAG64wRqhGmMKJ3+rZkqISAlcIvK6qqaIiAYwLmOMH9LT4csvXfuPuXNdacegQfDAA64qxhhjCjt/E5GxwBZgBTBXROoA1kbEmCA5ehTeew9eecVVuURHu1KQO+90jVGNMaao8CsRUdVRwCifRb+JSOfAhGSMyUlCghtw7K234MAB1+12yhRX9RJmfeCMMUWQXx9dIlIFeBqIAxSYD4wA9uexXWlgLlDKO9ZUVX1aROoCU4AqwFKgr6omn+6TMOZc99NPrvplyhQ3HHuPHq4LbocOIBLs6Iwx5vT521h1CrAXuAG40fv/Yz+2SwIuU9WWQCvgKhG5CBgJvKKqDYADwJ35DdyYc116Onz9NXTp4iacmzYN7r0XfvnFDUjWsaMlIcaYos/fRCRCVZ9V1c3e7TmgRl4bqZPo3S3h3RS4DJjqLX8P1wjWGAMcOwZjx0JsrBtyff16GDkStm1zw7DXrx/sCI0xpuD4m4jMFJGbRCTEu/UCvvVnQxEJFZGfgD3Ad8BG4KCqpnqrbAdOmd9TRPqLSLyIxO/du9fPMI0punbtgmHDXMPTe+6BcuVg8mQ3GNljj0GlSsGO0BhjCp6o5t0LV0SOAOWAdG9RCHDU+19VNdyPfZwHfAb8HZjoVcsgIrWBb1S1eU7btmvXTuPj4/OM05ii6OefXe+XyZMhJQWuuQYefthNRGdVL+ZMiMhSVW0X7DiMyY2/vWbOeCxGVT0oIrOBi4HzRCTMKxWpBew40/0bU5SowsyZrsvtd99BmTJw111u5NNGjYIdnTHGnD1+d/gTkWuBP3l356jqV35sUw1I8ZKQMsAVuIaqs3GNXqcAtwFf5DdwY4qiEydcycfLL8OaNW749eefh/79oUqVYEdnjDFnn7/dd18ALgAme4seFJGOqvpEHptGAO+JSCiuOucTVf1KRNYAU0TkOWA5MP70wjemaNizB958E8aMgb17oWVLNyDZTTdByZLBjs4YY4LH3xKRbkArVU0HEJH3cAlEromIqq4EWmezfBPQPn+hGlP0rFnjZrydNAmSkuDqq934H507W/sPY4yBfFTNAOcBv3v/2yDSxuRAFb7/3lW/fPMNlC4Nt90GgwdDkybBjs4YYwoXfxORfwLLvcamgmsr8njAojKmCEpKgo8+cgnIzz9D9eowYoTrilutWrCjM8aYwsnfXjMficgcXDsRgL+p6q6ARWVMEbJ/v5v75fXX3VggzZvDhAlw882uNMQYY0zO/G2sKkAXoJ6qjhCRaBFpr6pLAhueMYXX+vWu/cd778Hx43Dlle7/K66w9h/GGOMvf6tm3sANZnYZbrK7I8C/+aOExJhiQRX++183/sdXX7keL337wkMPuZIQY4wx+eNvInKhqrYRkeUAqnpARKzToSk2kpPhk09c+4/ly6FqVXj6aTcJXY08Z10yxhiTE38TkRRvLBCFzIHK0nPfxJii78ABNwHd6NGQkOB6vbz9NtxyixsN1RhjzJnxNxEZhZsnprqI/AM3KurfAxaVMUH2669uptsJE9xsuF26wLhxcNVVEOLvVJHGGGPy5G+vmckishTXYFWA61V1bUAjM+YsU4X58131yxdfQFgY9O7txv9o2TLY0RljzLnJ314zd6rqeGCdz7IXVNXGEjFFXkoK/PvfrgFqfDxUrgxPPgn33efmgjHGGBM4/lbN3CAiJ1R1MoCIjAFshARTpB08CO+8A6NGwbZt0LAhvPEG3HorlCsX7OiMMaZ48DsRAaaLSDpwFXBQVe8MXFjGBM7mza79x/jxkJgInTq5yeiuvtrafxhjzNmWayIiIpV97t4FfA4sAJ4Rkcqq+nv2WxpT+Cxa5Np/TJvmEo6//tW1/2jbNtiRGWNM8ZVXichSXJdd8fl7tXdToF5AozPmDKWmwmefuQTkxx/hvPPg0Ufh/vuhVq1gR2eMMSbXRERV656tQIwpSIcPu6qX116D336DevXcWCC33w7lywc7OmOMMRn8bSNiTJGwdatrfDpunEtG4uLglVfg2mshNDTY0RljjMkqYE3zRKS2iMwWkTUislpEHvSWDxeRHSLyk3frFqgYTPGxZAncdJMr+Xj1VejWDRYvhnnzoEcPS0KMMaawyquxaglVTTnNfacCD6vqMhGpACwVke+8x15R1X+d5n6NASAtDaZPd+0/5s+H8HA3+dwDD0B0dLCjM8YY44+8qmYWich24D/Af1R1i787VtWdwE7v/yMishaIOt1AjcmQmAjvvutKPjZtgjp1XPXLHXe4ZMQYY0zRkWvVjKq2Ax7y7r4qIv8TkVdEpKuIlPL3ICISA7QGFnuL7heRlSIyQUQq5bBNfxGJF5H4vXv3+nsocw7bvh3+9jeoXduVelSvDp9+6uaFeeghS0KMMaYoElX1f2WREsAluEHNOgF7VfXqPLYpD/wX+IeqThORGsA+XPffZ4EIVb0jt320a9dO4+Pj/Y7TnFt273YJyOTJkJ4Of/kLDBkCF18c7MiMKdxEZKn3g9KYQitfvWa89iI/eDdEJNeqFi9x+TcwWVWnefvY7fP4OOCrfMZsiglVeP99V9px7Jib++XBB6GudSo3xphzxhl131XVHTk9JiICjAfWqurLPssjvPYjAD2AVWcSgzk3bd0KAwbAf/4DHTq4MUGaNAl2VMYYYwpaIMcR6Qj0BX4WkZ+8ZU8CN4tIK1zVzBZgQABjMEVMejq89ZarilF1Y4IMHGjdb40x5lwVsEREVefjhoTPakagjmmKtl9+gbvugrlz4fLL3aBkMTHBjsoYY0wg+ZWIiEgj4FGgju82qnpZgOIyxUhqqut+O2wYlCrlqmH69QPJLo01xhhzTvG3RORT4C1gHJAWuHBMcbNyJdx5J8THw/XXw5gxEBkZ7KiMMcacLf4mIqmq+mZAIzHFSlISPP+8u1WqBB9/DD17WqXp5KcAACAASURBVCmIMcYUN3kN8V7Z+/dLERkIfAYkZTyuqr8HMDZzjlq82JWCrF4Nt9ziRkitUiXYURljjAmGvEpEluJ6t2T8Tn3U5zEF6gUiKHNuOnYM/v53l3hERsLXX7vJ6YwxxhRfuSYiqmpDR5kCMXu26xGzaRPccw+MHGlDshtjjMljrpkMInKfiJznc7+SV1VjTK4OHXIDk112mWv/MXs2vPmmJSHGGGMcvxIR4G5VPZhxR1UPAHcHJiRzrvjqK2jWDN55Bx55xPWQ6dQp2FEZY4wpTPxNREK9IdsBEJFQoGRgQjJF3b590KcPXHON6xGzaBG8+CKULRvsyIwxxhQ2/iYi3wIfi0gXEekCfAT8J3BhmaJIFaZMgaZN4dNP4emnYelSaN8+2JEZY4wprPwdR+QxoD9wr3f/O+CdgERkiqSEBLj3Xpg+HS64wI2Oev75wY7KGGNMYZdnIuJVw0xS1T640VWNyaQKEybAww+7Qcr+9S948EEIC+R0isYYY84ZeX5dqGqaiNQRkZKqmnw2gjJFw6ZN0L8/fP89XHqpa5TaoEGwozLGGFOU+Pu7dROwQESmA0czFqrqywGJyhRqaWkwejQ89RSEhsJbb8Hdd0OIvy2OjDHGGI+/ichG7xYCVAhcOKawW7vWDc++aJEbFfWtt6B27WBHZYwxpqjyKxFR1WcARKS8dz8xr21EpDYwCaiBGw7+bVV9zZu/5mMgBtgC9PLGJTGFWEoK/N//wYgRUL48vP++66Jrk9QZY4w5E/6OrNpcRJYDq4HVIrJURJrlsVkq8LCqxgIXAfeJSCzwOPC9qjYEvvfum0Js2TLXE2boULj+elcqcsstloQYY4w5c/7W6r8NDFHVOqpaB3gYGJfbBqq6U1WXef8fAdYCUcB1wHveau8B159O4CbwTpyAJ55w44Ds3g2ffQYffwzVqwc7MmOMMecKf9uIlFPV2Rl3VHWOiJTz9yAiEgO0BhYDNVR1p/fQLlzVjSlk5s93bUE2bIB+/eCll9woqcYYY0xB8rdEZJOI/F1EYrzbUFxPmjx57Ur+DTykqod9H1NVxbUfyW67/iISLyLxe/fu9TNMc6YSE2HQIPjTn9y4IDNnunFCLAkxxhgTCP4mIncA1YBpuKSiKtAvr41EpIS3/mRVneYt3i0iEd7jEcCe7LZV1bdVtZ2qtqtWrZqfYZozMXMmNG8OY8a4ZGTVKrjiimBHZYwx5lzmbyJyuao+oKptVLWtqj4E5PoV5U2SNx5Ym2W8kenAbd7/twFf5DdoU7AOHHDVL1deCaVLw7x58NprrneMMcYYE0j+JiJP+LnMV0egL3CZiPzk3boBLwBXiMgvwOXefRMk06ZBbKzrjvvkk/DTT9CxY7CjMsYYU1zk2lhVRP4MdAOiRGSUz0PhuO65OVLV+UBOHTy75CdIU/B274b774epU6FVK5gxA1q3DnZUxhhjipu8es0kAPHAtcBSn+VHgMGBCsoEjip88AE89JBrmPqPf8Cjj0KJEsGOzBhjTHGUayKiqiuAFSJyAPhKVdPPTlgmELZuhXvugW++gQ4dYPx4aNIk2FEZY4wpzvxtI9IL+EVE/k9E7KuriElPhzffhGbN4L//dQ1R5861JMQYY0zw+ZWIqOotuAHJNgITRWSRN86HTYBXyP3yC3TuDAMHwkUXuS65DzzgZs01xhhjgs3vidu9wcimAlOACKAHsExEBgUoNnMGUlPhxRehRQtYscJVw8ycCXXrBjsyY4wx5g9+DfEuItfiBjBrgJtRt72q7hGRssAaYHTgQjT5tXKlG549Ph6uuw7eeAMiI4MdlTHGGHMqf+eauQF4RVXn+i5U1WMicmfBh2VOR1ISPP+8u1Wq5Cao69nTZsk1xhhTePmViKjqbSJS0ysZUeB/qrrLe+z7QAZo/LN4sSsFWb0abrkFXnkFqlYNdlTGGGNM7vxqI+KVeiwB/gLcCPwoIncEMjDjn2PH4OGHXXfcQ4fgq6/cKKmWhBhjjCkK/K2aeQxorar7AUSkCrAQmBCowEze5syBu+6CjRthwAD4v/+D8PBgR2WMMcb4z99eM/txo6lmOOItM0Fw6JBLPDp3dvdnz4a33rIkxBhjTNHjb4nIr8BiEfkC10bkOmCliAwByDK7rgmgr792ScjOna5KZsQIKFs22FEZYwIhXdM5kXqCsiXsTW7OXf4mIhu9W4YvvL82oNlZsm+fmx9m8mQ3Quq0adC+fbCjMsacrsNJh0k4ksCOwztIOJLg/j+y46S/O4/s5NrG1zK119Rgh2tMwPjba+aZQAdisqfquuEOGuSqZJ5+Gp58EkqWDHZkxpjsJKUmsTNxZ2aCkZFUZP0/MTnxlG3DS4UTVSGKyAqRdIrpRGT5SNpGtg3CszDm7PG3RMQEQUIC3HsvTJ8OF1zgRkc9//xgR2VM8ZSWnsaeo3tOLb04vIOExITM//cfP7X5XKnQUkRWiCSyQiStaraiW4NuRFaIJCrcJR1RFaKIqBBB+ZLlg/DMjAkuS0QKIVWYMMG1AUlKckO1P/QQhNmrZUyBU1UOnjiYZ4KxK3EXaZp20rYhEkKNcjWIrBBJzHkxdKjVITO5yEgwIitEUrlMZcRGFjQmWwH9ahORCUB3YI+qNveWDQfuBvZ6qz2pqjMCGUdRsnkz3H03fP89/OlP8M470LBhsKMypmg6nnL81OoRL8HwbZtxPPX4KdtWKl0pM6mIrRabmVRk/I2sEEmN8jUIC7FfCMacCX/nmmkEvAnUUNXmItICuFZVn8tj04nA67j5aXy9oqr/ym+w57K0NHj9ddf+IzQU3nwT+veHEL+nJTSm+EhNT2V34u6TSy+OJJySYBw4ceCUbcuElclMMC6IuiDbBCOyQiRlSpQJwjMzpvjxN5UfBzwKjAVQ1ZUi8iGQayKiqnNFJOZMAiwO1q51w7MvWgR//jOMHQu1awc7KmPOPlVl//H9J5deZNOjZHfibhQ9adtQCSWiQgSRFSJpVKWRa+zpk2BkJB8VS1W0ahJjChF/E5Gyqroky5s39QyOe7+I3ArEAw+r6ik/W0SkP9AfIDo6+gwOVXilpLjRUEeMgPLl3dDsffrYJHXm3JSYnHhKgpFdj5LktORTtq1atmpmUtGqZqtsE4xqZasRGhIahGdmjDkT/iYi+0SkPm4wM0TkRmDnaR7zTeBZb1/PAi8Bp8xbo6pvA28DtGvXTrM+XtQtW+ZKQX76yc2QO3o01KgR7KiMyb/ktGR2Je7KtvTC9//DSYdP2bZciXJEhUcRVSGKjrU7nlQ9kpFgRJSPoFRYqSA8M2PM2eBvInIfLiloIiI7gM3ALadzQFXdnfG/iIwDvjqd/RRVJ07AM8+4njDVqrmByXr0CHZUxpwqXdPZd2xfrgNuJRxJYM/RPadsWyKkRGZC0axaM7rW63pScpFRolGhlI2JaExx5++AZpuAy0WkHBCiqkfy2iYnIhKhqhmlKT2AVae7r6Jm/nxXCrJhA/TrBy+9BJUqBTsqU9yoauaonjmVXuw4vIOdiTtJTT+5BlYQqpernplItI9sn22CUaVsFULEWlobY/Lmb6+Z84BbgRggLKOtiKo+kMd2HwGdgKoish14GugkIq1wVTNbgAGnF3rRkZgITzwBY8ZAdDTMnAlXXBHsqMy56ETqCXYe2ZnreBgJRxI4mnL0lG0rlqqYmVR0rtuZyPInD7gVWSGSmuVrUiK0RBCemTHmXOVv1cwM4EfgZyDd352r6s3ZLB7v7/bngpkzXTfcrVvh/vvh+eddw1Rj8sN3VM/cuqzmNqpnVHgUrSNa071R91MG3IqsEEm5kuWC8MyMMcWdv4lIaVUdEtBIzjEHDsCQITBxIjRuDPPmQceOwY7KFDYZo3qeMuBWliqT3Eb1jAqPom6lusRFx52SYESFR1GpdCXrrmqCaunSpdXDwsLeAZoDVmdXvKQDq1JTU+9q27btqQ3K8D8ReV9E7sY1LE3KWKiqv595jOeezz6DgQNh715XJTNsGJQuHeyozNl2LOVYnuNhJBxJ4ETqiVO2rVymcmZS0bx681NKL6LCo6herrqN6mmKhLCwsHdq1qzZtFq1agdCQkLOuV6QJmfp6emyd+/e2F27dr0DXJvdOv5+iiUDLwJPQeYoQgrUO+MozyG7d7tZcj/9FFq1gq+/hjZtgh2VKWip6ansStyVZ4Jx8MTBU7bNGNUzqkIUF0ZdmG2CEVE+wkb1NOea5paEFE8hISFarVq1Q7t27Wqe0zr+JiIPAw1UdV/BhHZuUYUPPnAT0yUmwj/+AY8+CiWsTV+R4juqZ24DbuU2qmdUhSgaV2lM55jOpwy4ZaN6mmIsxJKQ4st77XOskvM3EfkVOFYgEZ1jtm6Fe+6Bb76Biy+G8eOhadNgR2WySkxOzHM8jNxG9cxIKtrUbHPKgFtRFaKoWraqjeppjDGnwd9E5Cjwk4jM5uQ2Irl23z2Xpae7OWEee8z9/9prcN99bsI6c/YkpyVndlfNKcHYcXgHR5JPHfqmfMnymQlGXHQckeVPTTBqlq9po3oaUww9/vjjNV944YVdBbGvIUOGRJYvXz5txIgRu7N7vHfv3tG33377/qFDh9b617/+te1Pf/pTofvh/+abb1Z+5ZVXagKUK1cu/Y033vjt4osvPnXa6tPgbyLyuXczwC+/wF13wdy5cPnl8PbbULdusKM6N6kqG/ZvYNH2RWw5uOWUKdz3Htt7yja+o3o2r96crvW6njLgVmSFSBvV0xiTo1GjRkXkJxFJT09HVQk9jV+jy5YtKz9p0qSt+d7wLGrQoEHSggUL1lerVi3tk08+CR8wYECdlStXriuIffs7sup7BXGwoi41FV55xfWCKVXKVcP062eT1BWklLQUlu1cxvyt85m/bT4Lti7ITDYyRvWMCo+idsXaXBh1YbYJho3qaYzxtX79+pJXXXVVw/PPP//YqlWryjZq1Oj4p59+umX27NnlRo0aVX3WrFkbAT777LPwN954o1rDhg1PJCUlhTRp0iS2UaNGx6dPn755+PDhNSZPnlwVoG/fvnuHDRu2Z/369SWvvPLKRq1bt078+eefy82YMeOXlStXlh42bFhUWlqaVK5cOXXRokUbANauXVumffv2jRMSEkrec889u4cOHboHYNmyZaXr1at3Iizsj6/jtLQ0evXqFRMVFZU8atSohGnTpoWPGDEiMjk5WerUqZM0ZcqULRUrVkz/4osvKjz++OO109LSaNmy5bFJkyb9VqZMGY2Kijr/mmuuOfDDDz+ElypVSj/66KNNzZs3T5owYUKlf/7zn5EhISFaoUKFtPj4+PX+nsMrrrgicxTEzp07H73//vtLFtDLk3siIiKfqGovEfkZOKWhkaq2KKhACruff4Y77oD4eLjuOnjjDYiMDHZURd/hpMP8uP1Hl3hsnc+P23/keKor7atfqT7dGnYjLjqOjrU70qByAxvV05gi7o47qL1qFWULcp/Nm3NswgS25bbOli1bSo8dO3ZL165dj/bs2TPmxRdfrDZ8+PDdDz74YHRCQkJYZGRk6oQJE6r069dvX+/evQ9NnDix+rp169YAzJs3r+yHH35YZenSpWtVlbZt2zbt0qXLkapVq6Zt3bq11Pjx4zd36dJlS0JCQtj9998fM2fOnHVNmjRJ3r17d2bxyK+//lp64cKF6w8ePBjatGnT5o8++ujeUqVK6fTp0yt27dr1UMZ6KSkpcv3119eNjY09PnLkyF07d+4Me/755yPmzp27ITw8PP2pp56q+eyzz9YYMWLErgEDBtSdOXPm+hYtWiT16NEj5sUXX6w2bNiwPQAVK1ZM3bBhw5rXX3+9yqBBg2rPnj371xdeeCFi5syZG+rWrZuyb9++025IMHr06KqdO3c+lPea/smrRORB72/3gjpgUZOc7HrBPP+8mxdmyhTo1ctKQU5XwpGEzKRj/tb5rNi9gnRNJ0RCaF2zNf3b9s9MPCIqRAQ7XGPMOaJmzZrJXbt2PQrQt2/f/aNGjaoeEhKyu1evXvvHjRtX+b777tu/bNmy8tOmTducdds5c+aU79at28Hw8PB0gKuvvvrA7NmzK/Ts2fNgREREcpcuXY5665Vr3779kSZNmiQD1KhRI3MUwq5dux4sU6aMlilTJrVy5cop27dvD6tfv37KrFmzwj/44IMtGesNHDiwzvXXX//7yJEjd2Xsc+PGjaXbt2/fBFyi0rZt28QVK1aUrlWrVlKLFi2SAG6//fb9Y8aMqQ7sAbjtttt+B7j77rt/Hzp0aG2Adu3aJfbp0yfmhhtuONCnT58Dp3Mev/zyywoffPBB1YULFxZItQzkkYj4TE43UFX/5vuYiIwE/nbqVueOJUtcKcjq1dCnD7z6KlStGuyoig5VZd2+dczfOp95W+cxf+t8Nh907/GyJcpyUa2LGHrJUOKi47io1kXWZsOYYiCvkotAydptPuP+vffeu//qq69uULp0ab3mmmsOlMjnuAtly5b1a9qTUqVKZdYqhIaGkpqaKkeOHAk5fPhwaExMTErGY+3atUucN29e+LFjx3aXLVtWVZW4uLjDX3755UkJ0qJFi3IdbCgk5I/qaRFRgA8//HDrDz/8UG769OkV27ZtG7t06dI1NWvWzEyWBg0aFPXdd99VBMgoDfK1ePHiMgMHDqzz9ddf/+K73ZnytyI9uyna/lxQQRQ2x47BI4+47rgHD8JXX7lxQiwJyV1yWjKLti3ixQUvct2U66j2YjVi34il/1f9+Xbjt7SOaM3LXV9myV1LOPi3g3x/6/c80/kZrqh/hSUhxpiA2rlzZ8lZs2aVA5g8eXLlDh06JALExMSk1KhRI+Wll16K6N+/f+ZYWWFhYZqUlCQAnTt3TpwxY8Z5XuIQMmPGjEqdO3c+pStep06dji5ZsqTCunXrSgL4Vs1k5+uvv64QFxd30n4GDBiwr2vXroe6d+9ePyUlhU6dOh2Nj48vv2rVqlIAhw8fDlm5cmWpli1bntixY0fJjOWTJk2qcskll2Tua9KkSZUBxo8fX6l169ZHAVavXl3qsssuO/rqq68mVKpUKXXTpk0ntfMYPXr0jnXr1q3JLgn55ZdfSvbs2bP+hAkTNmeUwhSUvNqI3AsMBOqJyEqfhyoACwoykMJizhzXI2bjRhgwAEaOhIoVgx1V4XTwxEEWbVuU2bB0yY4lmcOVN6zckOsaX0dcdBxx0XE0qNzABvIyxgRNTEzMidGjR1fv379/2YYNG5545JFHMrvc3XTTTfvHjBkT1qZNm8z5Fvr06bO3adOmsc2bNz82ffr0zb17997fpk2bpuAaq3bs2PH4+vXrT/oij4yMTB01atSWHj16NEhPT6dKlSopCxcu/CWnmGbMmFGxV69ep1SRDB8+fPfgwYND//KXv9T9/PPPN48dO3bLTTfdVC85OVkAnn766R0tWrRIeuutt7b07NmzfkZjVd/ndODAgdBGjRrFlixZUqdMmbIJYPDgwbW2bNlSSlUlLi7u8EUXXeR399uhQ4dGHDx4MGzQoEF1wCVqq1atWuvv9rkR1ZwHuxORikAl4J/A4z4PHTmb88y0a9dO4+PjA3qMQ4fcmCBvvw3168O4cdC5c0APWeRsP7ydeb/Ny0w8ft79M4oSKqG0iWiTmXR0rN2RGuVrBDtcY4o9EVmqqu2CHceKFSu2tGzZMmgjc69fv75k9+7dG/7yyy+rs3v81ltvjW7duvWxwYMHn9UYY2Njmy5fvnydb7VNQYiKijo/Pj5+bURERGpB7vdMrFixomrLli1jsnssrzYih4BDwM0BiKvQ+PprV/qxcyc8/DCMGAFlC7RNd9GTrums2bvmpIalvx36DXADgV1c62Ju6HQDcdFxXBh1oU0hb4wpkpo1a9a0TJky6WPHjj3rbVfWrFlTICUKRV2xnrpz3z43P8zkydCsGUybBu3bBzuq4DiReoL4hPjMpGPBtgWZk7bVLF+TS6IvYcjFQ4iLjqNFjRY266sxpsho3Lhxck6lIatXrz7nkoEdO3b8HOwY8iOg3yYiMgHX9XePqjb3llUGPgZigC1AL1U9rW5Ep0sVPvnEzZR74AA8/TQ8+SSULLDhWQq/A8cPsHDbwsweLf9L+F/mPCtNqjbhxqY3EhcdxyV1LqHueXWtfYcxxpiACPTP2onA68Akn2WPA9+r6gsi8rh3/6x1A05IgIED4YsvoF07+P57OP/8s3X04FBVth7a+kc1y7b5rNqzCoCwkDDaRbbjgfYPEBcdR4faHahWrlqQIzbGGFNcBDQRUdW5IhKTZfF1QCfv//eAOZyFREQVJkxwbUCSkuDFF121TNg5WMOQlp7Gqj2rMpOO+Vvns/3wdgDCS4XToXYHbmp2E3HRcVwQdQFlSxTzBjHGGGOCJhhfwzV8BkrbBWTbvUJE+gP9AaKjo8/ogJs3Q//+MGsW/OlP8M470LDhGe2yUDmecpz/JfzP9WjZNp+F2xZyOOkwAJEVIrkk+hJXzRJ9Cc2rN7fp6o0xxhQaQZ0ZTF3f4Wy7Lanq26raTlXbVat2elUFaWnw2mvQvDn8+KObH2b27KKfhOw7to/p66fz2HeP0WF8Byq+UJFLJ17K0NlD2XZoGzc3v5n3e7zP5gc3s33wdqbcOIX7299Py5otLQkxxhg/PP744zULal9DhgyJHDZsWI5jGvTu3Tt65syZ5dq3b9947ty5hbKIevny5aVbtWrVpGTJkm2yPpepU6eGx8TENI+Ojm7+5JNP5vu8BaNEZLeIRKjqThGJwBsXPxCWLHHVL1ddBWPHwhkWrASFqrL54OaTutGu3ecaeZcMLckFkRdk9mbpULsDlctUDnLExhhT9I0aNSrihRde2OXv+unp6agqoaH5/7G3bNmy8pMmTdqa7w3PourVq6e+9tprW6dOnVrJd3lqaiqDBw+O/vbbbzfUq1cvpWXLlk1vuOGGg23btj2R076yCkaJyHTgNu//24AvAnWgiy+GBQtgxoyik4SkpaexfOdyRi0eRa9PexH1chT1R9Xnts9v45PVn1C3Ul2ev+x55t4+l0OPH2L+HfN54fIX6N6ouyUhxhiTjfXr15esW7dus2uvvbZuvXr1ml111VX1jhw5EjJ9+vQKl19+ef2M9T777LPwK664ov7AgQOjkpKSQpo0aRJ77bXX1gUYPnx4jYYNGzZr2LBhsxEjRlTP2G9MTEzzHj16xDRq1KjZxo0bS06dOjU8Nja2aePGjWMvvvjiRhn7Xrt2bZn27ds3rlWr1vnPPfdc9Yzly5YtK12vXr0TYT4NFtPS0rjhhhtiHnjggUiAadOmhbdq1apJbGxs0z//+c/1Dh06FALwxRdfVGjatGlso0aNYnv27Blz/PhxATeg2T333FOrUaNGseeff37TjGHgJ0yYUKlhw4bNGjduHNuuXbvG+TmHUVFRqZdeeumxEiVKnFSLMWfOnHJ16tRJio2NTS5durT+5S9/+X3q1Knn5Wffge6++xGuYWpVEdkOPA28AHwiIncCvwG9AhlDhw6B3PuZO5p8lCU7lmQ2LF20bRFHkt10AdEVo+lctzNxtd2Ipc2qNyNEglqbZowxZ+SOL+6ovWrPqgKtfmhevfmxCddNyHVAsi1btpQeO3bslq5dux7t2bNnzIsvvlht+PDhux988MHohISEsMjIyNQJEyZU6dev377evXsfmjhxYvWMOVfmzZtX9sMPP6yydOnStapK27Ztm3bp0uVI1apV07Zu3Vpq/Pjxm7t06bIlISEh7P7774+ZM2fOuiZNmiT7zjXz66+/ll64cOH6gwcPhjZt2rT5o48+urdUqVI6ffr0il27dj2UsV5KSopcf/31dWNjY4+PHDly186dO8Oef/75iLlz524IDw9Pf+qpp2o+++yzNUaMGLFrwIABdWfOnLm+RYsWST169Ih58cUXqw0bNmwPQMWKFVM3bNiw5vXXX68yaNCg2rNnz/71hRdeiJg5c+aGunXrpuzbt69A6um3bdtWMioqKjnjfq1atZIXL15cPj/7CHSvmZxGZO0SyOMWZnuO7mHB1gWZiceynctITU9FEM6vcT59W/R1w6RHdyS6YhEpxjHGmEKuZs2ayV27dj0K0Ldv3/2jRo2qHhISsrtXr177x40bV/m+++7bv2zZsvLTpk3bnHXbOXPmlO/WrdvB8PDwdICrr776wOzZsyv07NnzYERERHKXLl2OeuuVa9++/ZEmTZokA9SoUSNzhtquXbseLFOmjJYpUya1cuXKKdu3bw+rX79+yqxZs8I/+OCDLRnrDRw4sM7111//+8iRI3dl7HPjxo2l27dv3wRcotK2bdvEFStWlK5Vq1ZSxgR0t99++/4xY8ZUx2vucNttt/0OcPfdd/8+dOjQ2uBm9u3Tp0/MDTfccKBPnz5ndfyu3JyDnVcLD1Vl44GNJ83PsmH/BgBKhZaifVR7Hu3waGb7jvNK56s0yxhjipy8Si4CJeugjBn377333v1XX311g9KlS+s111xzoESJEvnab9myZdP9Wc93PpnQ0FBSU1PFm803NCYmJiXjsXbt2iXOmzcv/NixY7vLli2rqkpcXNzhL7/88qQEadGiRWVyO15IyB+l5yKiAB9++OHWH374odz06dMrtm3bNnbp0qVratasmZksDRo0KOq7776rCJDdDLzZqV27dvKOHTsyhwPdvn37SSUk/rBy/gKUmp5KfEI8r/74Kjd+ciMRL0XQcHRD7ph+B5+t+4zGVRoz8vKRLLhjAYceP8TcfnN5vsvzdGvYzZIQY4wJoJ07d5acNWtWOYDJkydX7tChQyJATExMSo0aNVJeeumliP79+2dOehcWFqZJSUkC0Llz58QZM2ac5yUOITNmzKjUuXPnI1mP0alTp6NLliypsG7dupIAvlUz2fn6668rxMXFnbSfAQMG7OvateuhfodQCwAAF6lJREFU7t27109JSaFTp05H4+Pjy2e08zh8+HDIypUrS7Vs2fLEjh07SmYsnzRpUpVLLrkkc1+TJk2qDDB+/PhKrVu3PgqwevXqUpdddtnRV199NaFSpUqpmzZtOmk88dGjR+9Yt27dGn+TEIBLL7306JYtW0qvW7eu5IkTJ2TatGmVb7jhhoP+bg9WInJGEpMT+XH7j5m9WX7c/iNHU44CUPe8unSt3zVzRtomVZtY+w5jjAmSmJiYE6NHj67ev3//sg0bNjzxyCOP7M147Kabbto/ZsyYsDZt2mT29OjTp8/epk2bxjZv3vzY9OnTN/fu3Xt/mzZtmgL07dt3b8eOHY+vX7/+pC/yyMjI1FGjRm3p0aNHg/T0dKpUqZKycOHCX3KKacaMGRV79ep1ShXJ8OHDdw8ePDj0L3/5S93PP/9889ixY7fcdNNN9ZKTkwXg6aef3tGiRYukt956a0vPnj3rp6Wl0bJly2O+z+nAgQOhjRo1ii1ZsqROmTJlE8DgwYNrbdmypZSqSlxc3OGLLrrouL/nb+vWrWEXXHBB7NGjR0NFRMeOHVtj7dq1qypXrpz+0ksvbb3qqqsapaWl0bt3733t2rXzu8cMgLihPAq3du3aaXx8fLDDYFfirpO60f606yfSNA1BaFmzZWaj0o7RHakVXivY4RpjijkRWaqq7YIdx4oVK7a0bNlyX95rBsb69etLdu/evWFOE9/deuut0a1btz42ePDgsxpjbGxs0+XLl6/zrbYpCFFRUefHx8evjYiISC3I/Z6JFStWVG3ZsmVMdo9ZiUgOVJUN+zecNEz6r7//CkCZsDJcWOtCnoh7grjoOC6qdREVS1cMcsTGGGPyq1mzZk3LlCmTPnbs2LPedmXNmjXn3My/p8MSEU9yWjLLdy4/KfHYd8wlx1XLViUuOo572t5DXHQcrSNaUzK0GE3Va4wxRVjjxo2TcyoNWb169TmXDOzYsePnYMeQH8U2ETmcdJgft/+YOT/L4u2LOZ7qqsvqV6rP1Q2vzpyfpVGVRqe0uDbGGGPMmSs2iUjCkYST2nes2L2CdE0nREL+v717j66iuvcA/v0lIQkJSSABQiCEEzSvAxIgaQoYLS9TEKQCDUW4ivZaVJTaULz1cUspbb1axGVR1xUt1MutlF4RNdK0AjUssFhsAkTDI8gjRgJIQF4GyOv87h+zTzjEBDjkMXl8P2vNypw9c/b8zszA/M6efWZjSK8hmJ0y2+rf0fdmRIVE2R0uERFRh9CuE5GDpw5i4aaF+LDkQxw6bf0EO6hTEIZHD8fPb/050mPS8e0+30ZIQIjNkRIREXVM7ToRCfANwPsH3kd6TDrmps1Fekw6BvcajE6+3j2whoiIiJpHu36wRZ/QPjj202N4a9pbyBqehW/1+RaTECIiuqrHH3/c6+HsGzJv3rzeCxYsiGxo+YwZM2LWr18fnJaWlrB58+YmHYenqaxbty4kJCRkcGJiojMxMdE5f/782j4Ma9asCXU4HANjYmIGPvnkk17vt3adiADffKwvERHR1SxdutSrzoIulws1NTVXX7Ee27dv7zJ69Ojy63pzC0pNTf3a/eTV55577igAVFdXIysrKyYnJ2ffvn37dr311lvh+fn5gd7U2+4TESIi6tiKior8Y2NjB0yaNCm2f//+A8aNG9f/3LlzPtnZ2SFjx469wb3e22+/HXrbbbfdMGfOnD4VFRU+iYmJzkmTJsUCwMKFCyPj4uIGxMXFDVi0aFFPd70Oh2Pg5MmTHfHx8QMOHDjgv2bNmlCn05mUkJDgHD58eLy77j179nROS0tLiI6OvunXv/51T3f59u3bA/v373/Rz+9ST4mamhpMnTrV8eMf/7g3AKxduzZ08ODBiU6nM2n8+PH9z5w54wMA7777bkhSUpIzPj7emZmZ6bhw4YIA1gPNHnzwwej4+HjnTTfdlOR+DPyKFSu6xcXFDUhISHCmpqYmNMW+3bRpU3C/fv0qnE5nZWBgoE6ZMuWrNWvWeDVmSbvuI0JERK3MD3/YF4WFTXv7YeDA81hx5cH0iouLA5ctW1ackZFRnpmZ6Vi8eHGPhQsXfvnoo4/GHDlyxK93797VK1asiLjvvvtOzJgx48zrr7/e0z3mypYtW4JWrVoVkZ+fv0dVkZKSkjRmzJhz3bt3rykpKQlYvnz5oTFjxhQfOXLE75FHHnFs2rRpb2JiYqXnWDP79+8P3Lp1a9Hp06d9k5KSBj722GNlAQEBmp2dHZaRkXHGvV5VVZXceeedsU6n88Kzzz577OjRo35PP/101ObNm/eFhoa6nnrqqV6/+tWvIhctWnTsgQceiF2/fn3RoEGDKiZPnuxYvHhxjwULFhwHgLCwsOp9+/btfumllyLmzp3bNzc3d/8zzzwTtX79+n2xsbFVJ06cuOI4OPXZsWNHl4SEBGdkZGTV888//0VqaurFL7744rJB7qKjoyu3bdvWxZt62SJCRETtXq9evSozMjLKAeDuu+8+uXXr1i4+Pj6YNm3ayddeey38xIkTvtu3b++SmZl5pu57N23a1OX2228/HRoa6goLC3NNmDDhVG5ubggAREVFVY4ZM6bcrBeclpZ2LjExsRIAIiMja+/VZGRknO7cubNGRUVVh4eHVx0+fNgPADZu3Bh65513nnWvN2fOnH7uJMRd54EDBwLT0tISExMTnatXr44oKSnxLygoCIyOjq4YNGhQBQDce++9Jz/88MPan4DOmjXrKwD40Y9+9NWOHTu6ANatlZkzZzqWLFnSvbrau6e/jxgxovzzzz//pKioaPfDDz98fOrUqTd6VcEVsEWEiIhazlVaLppL3f6C7tcPPfTQyQkTJtwYGBiod9xxx6lOnbz7QUNQUJDrWtbzHE/G19cX1dXVYkbz9XU4HFXuZampqV9v2bIl9Pz5818GBQWpqiI9Pf3se++9d8izvo8++qjzlbbn43OpnUFEFABWrVpV8sEHHwRnZ2eHpaSkOPPz83f36tWrNlmaO3dunw0bNoQBQN0ReMPDw2s/5w9+8IMz8+bNizl69Khf3759K0tLS2sfNX748OHLWkiuhW0tIiJSLCKfishOEbF/RDsiImq3jh496r9x48ZgAHjjjTfCR4wY8TUAOByOqsjIyKolS5ZEzZ49u3bQOz8/P62oqBAAGDVq1Nc5OTldTeLgk5OT023UqFHn6m5j5MiR5R9//HHI3r17/QHA89ZMff7yl7+EpKenX1bPAw88cCIjI+PMxIkTb6iqqsLIkSPL8/Lyurj7eZw9e9bnk08+CUhOTr5YWlrq7y5fuXJlxC233FJb18qVK8MBYPny5d2GDBlSDgC7du0KGD16dPkLL7xwpFu3btUHDx68bKySF198sdTdGbVurCUlJX4ul5WL5ObmBrlcLkRGRlZ/5zvfKS8uLg7cu3ev/8WLF2Xt2rXhU6dOPX2lz12X3S0io1TVthEZiYioY3A4HBdffPHFnrNnzw6Ki4u7OH/+/DL3sunTp598+eWX/YYOHVo7fP3MmTPLkpKSnAMHDjyfnZ19aMaMGSeHDh2aBAB333132c0333yhqKjosgt57969q5cuXVo8efLkG10uFyIiIqq2bt36WUMx5eTkhE2bNu1U3fKFCxd+mZWV5TtlypTYd95559CyZcuKp0+f3r+yslIA4Be/+EXpoEGDKl555ZXizMzMG2pqapCcnHze8zOdOnXKNz4+3unv76+rV68+CABZWVnRxcXFAaoq6enpZ4cNG3bhWvffH//4x24rVqzo6evrq4GBga6VK1ce9PHxgY+PD5YsWVIybty4+JqaGsyYMeNEamrqxavXeImoNunow9e+YZFiAKnXkoikpqZqXh4bTYiIvCEi+aqaanccBQUFxcnJybZ96SwqKvKfOHFiXEMD391zzz0xQ4YMOZ+VldWiMTqdzqQdO3bs9bxt0xT69OlzU15e3p6oqCjvOoI0o4KCgu7JycmO+pbZ2SKiANabe1fLVPVVz4UiMhvAbACIiYmxITwiImrvBgwYkNS5c2fXsmXLWrzvyu7du9vdyL/Xw85EJF1VS0WkJ4ANIrJXVTe7F5rE5FXAahGxK0giImrbEhISKhtqDdm1a1e7SwZKS0s/tTsGb9jWWVVVS83f4wDeBpBmVyxERNSsXC6Xi4+57qDMsW/w10W2JCIiEiwiIe55ABkACu2IhYiIml1hWVlZGJORjsflcklZWVkYrnCNt+vWTCSAt83vuP0ArFLVv9kUCxERNaPq6ur7jx079vtjx44NBB+k2dG4ABRWV1ff39AKtiQiqnoQQLId2yYiopaVkpJyHMAku+Og1omZKREREdnG7geaERG1T6pAdXXjp4gIYOhQuz8NUbNhIkJEzcPlAmpqmuZi3BYn1zUNQXJ1d9wBZGc3TV1ErRATEaLmwgux3UcA8PNr3BQU1Pg6GjtFRNi9F4maFRMRah1Ugaoq4Pz5y6fy8kvzVVX2X1y9mWwaPuEy7eFCfL2Tjw8g/LUoUWvHRISurqYGuHDhm0lCU081NVePxVuNvZgFBNh/QeWFmIjaMSYibZkqUFnZdImAZ+uD51RR4X1sPj5AcLD1jbru1L17/eVXmjp39j4p8OGPwoiIWjsmIs2loVaEhi721ztdz334wMD6L/bBwUCPHt4nCfXV06kTv40TEdFVdbxERNX6ht/ctxmauhXhSglCQ+9pqGWBLQVERNRKtO9EpKgImDz5m0nC9XQi9GxF8Lzwd+kC9OzZ+FaEoCC2IhARUYfTvhOR4GBgwIDGJwhsRSAiImoW7TsRiY4G3nzT7iiIiIioAfyaT0RERLZhIkJERES2YSJCREREtmEiQkRERLaxLRERkXEiUiQi+0XkcbviICIiIvvYkoiIiC+AlwGMB+AEcJeIOO2IhYiIiOxjV4tIGoD9qnpQVSsBrAbwPZtiISIiIpvYlYj0AfCFx+vDpqyWiMwWkTwRySsrK2vR4IiIiKhltNoHmqnqqwBeBQARKRORzxtRXXcAJ5oksKbFuLzDuLzDuLzTHuPq15SBEDUHuxKRUgB9PV5Hm7J6qWqPxmxMRPJUNbUxdTQHxuUdxuUdxuUdxkVkD7tuzfwLQJyIxIqIP4DpALJtioWIiIhsYkuLiKpWi8gjAN4H4AtgharusiMWIiIiso9tfURUNQdATgtt7tUW2o63GJd3GJd3GJd3GBeRDURV7Y6BiIiIOig+4p2IiIhsw0SEiIiIbNMmExER6SsiuSKyW0R2icijpjxcRDaIyGfmbzdTLiKy1Ixr84mIDPWoa5ZZ/zMRmdXIuAJF5GMRKTBx/dKUx4rINrP9P5tfCkFEAszr/Wa5w6OuJ0x5kYh8tzFxedTpKyI7RGRda4lLRIpF5FMR2SkieabM1uNo6usqImtEZK+I7BGR4XbHJSIJZj+5p7Mi8hO74zL1ZZlzvlBE/mT+LbSG8+tRE9MuEfmJKWvx/SUiK0TkuIgUepQ1WRwikmL+He0375Xr22NENlDVNjcBiAIw1MyHANgHa8ya3wJ43JQ/DuBZM387gL8CEADDAGwz5eEADpq/3cx8t0bEJQC6mPlOALaZ7f0fgOmm/BUAD5n5OQBeMfPTAfzZzDsBFAAIABAL4AAA3ybYb/MArAKwzry2PS4AxQC61ymz9TiaOv8HwP1m3h9A19YQl0d8vgCOwXpgld3nfR8AhwB09jiv7rX7/AIwEEAhgCBYHfM3ArjRjv0F4FYAQwEUNsd5DuBjs66Y945vivOME6eWmGwPoEk+BPAugNsAFAGIMmVRAIrM/DIAd3msX2SW3wVgmUf5Zes1MqYgANsBfBvWUxH9TPlwAO+b+fcBDDfzfmY9AfAEgCc86qpdrxHxRAP4O4DRANaZ7bSGuIrxzUTE1uMIIAzWhVVaU1x1YskA8I/WEBcuDdkQbs6XdQC+a/f5BSATwHKP1z8H8B927S8ADlyeiDRJHGbZXo/yy9bjxKm1T23y1own06w7BFbrQ6SqHjWLjgGINPMNjW1z1TFvriMeXxHZCeA4gA2wvtWdVtXqerZRu32z/AyAiOaIC8ALsP4TdpnXEa0kLgWwXkTyRWS2KbP7OMYCKAPwB7FuZf1eRIJbQVyepgP4k5m3NS5VLQXwHIASAEdhnS/5sP/8KgRwi4hEiEgQrJaGvmg9x7Gp4uhj5ps6PqIW0aYTERHpAuAtAD9R1bOey1RVYV3kWpSq1qjqYFgtEGkAEls6hrpEZCKA46qab3cs9UhX1aEAxgN4WERu9Vxo03H0g9WM/t+qOgRAOaymc7vjAgCYvhaTALxZd5kdcZm+Dd+DlcD1BhAMYFxLxlAfVd0D4FkA6wH8DcBOADV11rHtOLbGOIjs0GYTERHpBCsJeUNV15riL0UkyiyPgtUqATQ8to1XY954Q1VPA8iF1STdVUTcD4/z3Ebt9s3yMAAnmyGumwFMEpFiAKth3Z75XSuIy/1tGqp6HMDbsJI3u4/jYQCHVXWbeb0GVmJid1xu4wFsV9UvzWu74xoL4JCqlqlqFYC1sM651nB+LVfVFFW9FcApWP3J7N5fbk0VR6mZb+r4iFpEm0xETI/w5QD2qOrzHouyAbh7ks+C1XfEXX6P6Y0+DMAZ0yT6PoAMEelmvtVlmLLrjauHiHQ1851h9VvZAysh+X4Dcbnj/T6AD8w3o2wA082vC2IBxMHqjHZdVPUJVY1WVQesJv0PVHWm3XGJSLCIhLjnYe3/Qth8HFX1GIAvRCTBFI0BsNvuuDzchUu3ZdzbtzOuEgDDRCTI/Nt07y9bzy8AEJGe5m8MgCmwOmvbvb/cmiQOs+ysiAwz+/8ej7qIWj+7O6lczwQgHVYz5iewmlt3wrr/GwGrQ+ZnsHrIh5v1BcDLsPprfAog1aOuHwLYb6b7GhnXIAA7TFyFABaY8v6w/kPdD6s5PcCUB5rX+83y/h51PWXiLUIT9oAHMBKXfjVja1xm+wVm2gXgKVNu63E09Q0GkGeO5TuwfqXQGuIKhtV6EOZR1hri+iWAvea8/19Yv3yx/bwHsAVWUlQAYIxd+wtW4ngUQBWsFrd/b8o4AKSafX8AwEuo09GaE6fWPPER70RERGSbNnlrhoiIiNoHJiJERERkGyYiREREZBsmIkRERGQbJiJERERkGyYi1K6JSI1YI9UWiMh2ERlxlfW7isica6h3k4ikXmdMOe7nzRARdXRMRKi9u6Cqg1U1Gdagav91lfW7whodttmo6u1qPXmXiKjDYyJCHUkorMd8Q0S6iMjfTSvJpyLyPbPOMwBuMK0oi826PzPrFIjIMx71ZYrIxyKyT0RuqbsxEYkSkc2mrkL3OiJSLCLdReRBs2yniBwSkVyzPENEPjKxvSnWmEpERO0SH2hG7ZqI1MB6OmUgrOHSR6tqvhnjJEhVz4pIdwD/hPVI8X6wnjw70Lx/PKzh48eq6nkRCVfVr0RkE4B8Vf2piNwOYJ6qjq2z7Z8CCFTV34iIr9neObHG/ElV1RNmvU4APgDwWwAfwRqrZbyqlovIz2A9kXRRc+4nIiK7+F19FaI27YJaoyFDRIYDWCkiA2E9RvtpsUb7dcEaNj2ynvePBfAHVT0PAKr6lccy92CL+QAc9bz3XwBWmETjHVXd2UCMv4M13sp7Yo2U7ATwD2vYEPjDSk6IiNolJiLUYajqR6b1owessYl6AEhR1SrTShHoZZUV5m8N6vm3pKqbTaIzAcDrIvK8qq70XEdE7oXVCvOIuwjABlW9y8tYiIjaJPYRoQ5DRBIB+MIMGgfguElCRsFKBgDgHIAQj7dtAHCfiASZOsK92F4/AF+q6msAfg9gaJ3lKQDmA/g3VXWZ4n8CuFlEbjTrBItIvHeflIio7WCLCLV3nUXEfUtEAMxS1RoReQPAeyLyKaxRdvcCgKqeFJF/iEghgL+q6mMiMhhAnohUAsgB8OQ1bnskgMdEpArA17CGZ/f0CIBwALnmNkyeqt5vWkn+JCIBZr3/BLDP609ORNQGsLMqERER2Ya3ZoiIiMg2TESIiIjINkxEiIiIyDZMRIiIiMg2TESIiIjINkxEiIiIyDZMRIiIiMg2/w9FRNYAdctgzgAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plot_relative_time(results, n_features, max_batch_size=max_batch_size)" + ] + }, + { + "cell_type": "markdown", + "id": "b96a904b", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "The difference between KeOps and PyTorch is even more striking when we only look at $[2, 10]$ features:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "0d1e4dfa", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAEWCAYAAAAQHy/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdeVhV1frA8e9iEFGBRJFwQBxAUbQcU8uRQXPMzCEShyyzbllqdbv3dpt/TYqVXsuh1BwrLcspp0TFzJxnMSccUVFEQGU86/fHPigSICCwGd7P85xHzt777PXuc45nvXvttddSWmuEEEIIUbbYmB2AEEIIIYqeJABCCCFEGSQJgBBCCFEGSQIghBBClEGSAAghhBBlkCQAQgghRBkkCUAJoZTSSqn6OayfqpT6b4bnzyulLiqlEpRSVfJRnpe1TLv8xlxQrMdQtwjK8bSWZVsEZbkrpTYppeKVUqGFXZ7IPaXUv5VSX5sdhxCFTRKATJRSkUqpZKVU1UzLd1srRC/r89nW530ybfeZdfkw6/NhSqk0a8WSoJQ6qZSapZTyKci4tdajtNbvW8u0ByYCQVrrSlrrK3dLIO6F9Rg3F9C+Niilnsm4zHoMJwpi/5nKilRKBWQo57S1rLSCLisLI4HLgLPWety97kwp5aGUWqqUOp/xe5phvYNSaqZSKk4pdUEpNTbTen+lVIRS6oZSKkwpVfteYzKDUuodpdS8PGzfSSl1NuMyrfWHWutnsnuNEKWFJABZOwk8mf5EKdUEqJDFdn8BQzJsZwcMAI5n2u4PrXUlwAUIAG4CO5VSfgURbBZnrO5AeeBgQexfFIrawCGdj5G4smmVsQCrgH7ZvOwdwNtabmfgdaVUN+v+qgI/Af8FXIEdwPd5jSuvikPrkhBlmtZaHhkeQCTwJrA9w7IJwH8ADXhZl822Lr8IVLYu6wn8CmwGhlmXDQM2Z1HOcmBxDnG8BkQB54GnrWXXz1D2V8BK4DpGUjEb+ADwsS7TQAKwHthkfX7dumxgFuXZWo/nMnAC+If1NXbW9S7AN9aYzlnLsgV8gUQgzbrvWOv2Dtb9nba+R1MBxwzl9QH2AHEYCVM34P+s+0m07ut/1m0zHrsLMAeIBk5ZPyubjO+1tdyrGInco9m8v3MxKs2b1rJeB7wyHfMG63FusW6zDKgCzLfGvT39+2DdviGwFogBjgADsil7NpACJFv3G2B9vz63ft7nrX87WLfvBJwF/glcAObm8L2xI8P3NMPy8xgtQunP3we+s/49EtiSYV1F6/vSMIf/I/8CDlnf51lA+Qzre1o/21jre9c002v/CewDkqzxRmJ83/dhfEe/wUhifwXigXXc/j/WCTibRTwBGN+hZOt7mwDsta4fDhy27usE8Fym47RYt08AqmMkS/My7L83RjIda/1O+GYq+1Vr7NcwEqfy1nVVMf6fx1q/E+FYv6vykEdxeJgeQHF7ZPgxOYJRudlaf3xr8/cE4ANgOvC8ddkPGC0HuUkAngYuZhNDN4xK08/6I7WAvycA14CHMVpxyqfHY13vRYaKzLrs1uuzKXMUEAHUwjgLDOPOynAJMM0aTzVgW4Yf0r8dI/AZsNS6LyeMyvMj67rW1vgDrfHXwFrZWH9gn8m0r4zHPgf4xbpPL4xWmBEZ4kgBnrV+bs9jVHwqp886w/M73jdrLMeAehiJxyFreQEYFdccYJZ124rAGYzKxg5ohpFMNcqm7Fufl/X5e8BW63vrhlFxvm9d1wlIBT7BSBQcs9qnddu/JQBAZesy9wzLngD2W//+Avgq034OAP1yeN8OZPiu/M7t714z4BLwkPUzGGrd3iHDa/dYX+uYYdlWjEq/hvX1u6z7Ko+RxL6d4b3IMgGw/v0OGSpv67Ie1s9QAR2BG0DzHPZ3ax/cTqgDAXuMRPEYUC5D2dswEgdXjERjlHXdRxiJr7310Z5svovykIcZD7kEkL25GM37gRj/qc9ls90cYIhS6j6MH5efc7n/8xg/GFkZgFGxHNBaX8f4QcrsF63171pri9Y6MZdl5mQA8LnW+ozWOgbjxwswOqwB3YFXtNbXtdaXMCr4QVntSCmlMM4qx2itY7TW8cCHGbYfAczUWq+1xn9Oax1xtwCtlzoGAf/SWsdrrSOBUCAkw2antNYztHEd/1vAA6Niya9ZWuvjWutrGGekx7XW67TWqcAijEoKjLPeSK31LK11qtZ6N/Aj0D+X5TwFvKe1vqS1jgbezXRcFoxKMElrfTOPx1DJ+u+1DMuuYSRR6euvcaeM67Pyvwzflf/j9iWzkcA0rfWfWus0rfW3GGf6bTK8dpL1tRmPY7LW+qLW+hzGmfKfWuvd1u/2Em6/z3mmtV5h/Qy11nojsAajMs6NgcAK63c1BaN1yRFol+l4zlvfi2XAg9blKRjfv9pa6xStdbjWWiZfEcWGXIPL3lyMpvM6GJV8lrTWm5VSbhiXCJZrrW8a9d9d1cBoFsxKdWBnhuenstjmTG4KyYPqmfaZsczaGGcwURmOzSaHGNww+kzszLC9wjgjBOPsb2U+YqxqjSNjbKcw3st0F9L/0FrfsJZfify7mOHvm1k8T993beAhpVRshvV2GN+j3KjO34+reobn0feQ6CVY/3XGuLyS/nd8hvXOmV6TcX1WMn9X0mOtDQxVSr2UYX057jyWrL43uX2f80wp9SjwNsbZvA3Gd3N/Ll9+x+eitbYopc6QzXcOo3Uh/VjHYyTva6zfw+la64/zcQhCFAppAciG1voUxjXk7hgdpHIyDxhHDolCFvpinOlkJQqjkkznmVWIeSgrN3Iq8wzGWVxVrfV91oez1rpxNrFcxvjRbpxhexdtdIRM31+9bOLI6bguY5xVZeyh7kn2rTN3U5Dv4RlgY4bjvU8bdxQ8n8vXn+fvx3U+w/N8x6q1vorx+T6QYfED3O4kejDjOqVURYzPJ6dOpJm/K+mxngH+L9P7UEFrvTBjSPk7EsBojr/VIdfaKuSW3b6VUg4YLTETMC6B3IeRfKqsts/CHZ+LtXWrFrn4zllbqcZpreti9CMYq5Tyv9vrhCgqkgDkbATQxdoMn5NJGJcKNuW0kVLKVilVRyk1GePa47vZbPoDMEwp1UgpVQHj7OVeXQRyupf+B2C0UqqmUqoy8Eb6Cq11FEazaahSylkpZaOUqqeU6phh3zWVUuWs21uAGcBnSqlqAEqpGkqprtbtvwGGW289s7Gua3i3OK3N+j8A/6eUcrLeqjYWIwHLj7u9J3mxHPBRSoUopeytj1ZKKd9cvn4h8KZSys3aK/8t8nhcSqnyGH0EABysz9PNse6/svW9fhajHwIYTex+Sql+1te8Bey7y2WZf1i/K64YrV/pdw3MAEYppR5ShopKqR5KqZwuJ+TFX0B56z7tMTqBOmRYfxHwUkql/7aVs66PBlKtrQFBmbavopRyyaa8H4Ae1u+qPUain4TRRyNHSqmeSqn61qThGkYHV0tuD1SIwiYJQA6s1w135GK7GK31bzlc32urlErA6Dm+AaN5tZXWOstmSK31rxi9wNdjdDhan5/4M3kH+FYpFauUGpDF+hnAamAvRgeszK0eQzB+TNN7fi/GuL6JNb6DwAWl1GXrsn9aY9+qlIrD6MndwHp82zA6y32G8cO4kdtnWV8ATyilriqlJmUR50sYZ4EnMDpbLgBm5u4t+JuPMCrFWKXUq/ncB2Cc7WFULIMwzhovcLvTXm58gHH73T6M5uld1mV5kX5HAxgdOjNeY38b426LUxjv93it9Spr7NEYtw/+H8Zn+xDZ9O/IYAFGUnjCut8PrPvagZFc/M+6r2MYnTMLhLUvxgvA1xhn4dcxOummW2T994pSapf1cxmNUZFfBYIxOqem7y8CI/k6Yf0eZLxUgdb6CDAYmIzRAtUL6KW1Ts5FuN4Y3/sE4A/gS611WN6OWIjCo6RPihAiL5RSkRh3aqwzOxYhRP5JC4AQQghRBkkCIIQQQpRBcglACCGEKIOkBUAIIYQog0rEQEBVq1bVXl5eZochhBAlys6dOy9rrd3uvqUoi0pEAuDl5cWOHXe9G08IIUQGSqmsRhEVApBLAEIIIUSZVGgJgFJqplLqklLqQBbrximltHXEMyGEEEIUscJsAZiNMa3tHZRStTBGTDtdiGULIYQQIgeFlgBorTeR9Wx3n2HMqS33HwohhBAmKdI+AEqpPsA5rfXeoixXCCGEEHcqsrsArLPa/Zs7Z+LKafuRwEgAT8+sZsMVQgghRH4VZQtAPaAOsNc6mUhNYJdS6v6sNtZaT9dat9Rat3Rzk9tYhRBCiIJUZAmA1nq/1rqa1tpLa+2FMYVnc631haKKQQghSorrydd5+deXiU2MNTsUUUoV5m2ACzHmwG6glDqrlBpRWGUJIURpEpsYS9C8IP63/X/8fvp3s8MRpVSh9QHQWj95l/VehVW2EEKUVNHXowmaF8TBSwf54Ykf6OHTw+yQRClVIoYCFkKIsuBc3DkC5gYQGRvJ0ieX0q3+34ZSEaLASAIghBDFwImrJwiYE8DlG5dZPXg1HWp3MDskUcpJAiCEECY7FH2IwLmBJKYm8tuQ32hVo5XZIYkyQBIAIYQw0a6oXXSd1xVbZcvGYRvxq+ZndkiijJDZAIUQwiS/n/6dzt92poJ9BcKHh0vlL4qUJABCCGGCdSfWETQviPsr3c/m4ZvxruJtdkiijJEEQAghitgvEb/QY0EP6lWux6Zhm6jlUsvskEQZJAmAEEIUoQX7F9Dvh348eP+DbBi2AfdK7maHJMooSQCEEKKITN85ncE/DaZ97fasC1mHq6Or2SGJMkwSACGEKAKhW0J5bvlzPOr9KCuDV+Lk4GR2SKKMkwRACCEKkdaat8Pe5tW1r9K/UX+WDFyCo72j2WEJIeMACCFEYdFaM27NOD7b+hnDHxzOjF4zsLWxNTssIQBJAIQQolCkWdIYtXwUX+/+mtGtR/NZt8+wUdLoKooPSQCEEKKApaSlMOTnIXx34Dv+0/4/vN/5fZRSZoclxB0kARBCiAKUmJrIgEUDWPbXMj72/5h/PvJPs0MSIkuSAAghRAFJSE7gse8e47eTvzGl+xReaPWC2SEJkS1JAIQQogDEJsbSfX53/jz3J3Mem0PIAyFmhyREjiQBEEKIe3Tp+iW6zuvKwUsHWdR/EY/7Pm52SELclSQAQghxD87GnSVwbiCnYk+x7MlldK3f1eyQhMgVSQCEECKfTlw9gf8cf67cuMLqwatpX7u92SEJkWuFdlOqUmqmUuqSUupAhmXjlVIRSql9SqklSqn7Cqt8IYQoTIeiD/HIzEeIS4pj/dD1UvmLEqcwR6WYDXTLtGwt4Ke1bgr8BfyrEMsXQohCsStqFx1mdUCj2ThsIy2rtzQ7JCHyrNASAK31JiAm07I1WutU69OtQM3CKl8IIQrD76d/p/O3nalUrhLhw8Pxq+ZndkhC5IuZ41I+Dfya3Uql1Eil1A6l1I7o6OgiDEsIIbK29vhaguYFcX+l+wkfHk591/pmhyREvpmSACil/gOkAvOz20ZrPV1r3VJr3dLNza3oghNCiCz8HPEzPRf2pL5rfTYN20Qtl1pmhyTEPSnyBEApNQzoCTyltdZFXb4QQuTV/H3zeeKHJ2h2fzPChobhXsnd7JCEuGdFmgAopboBrwO9tdY3irJsIYTIj2k7phGyJIQOtTuwNmQtro6uZockRIEozNsAFwJ/AA2UUmeVUiOA/wFOwFql1B6l1NTCKl8IIe7VhC0TGLViFN29u7MieAVODk5mhyREgSm0gYC01k9msfibwipPCCEKitaatze8zfub3mdA4wHM7TuXcrblzA5LiAIlIwEKIUQGWmvGrh7L539+ztMPPs30XtOxtbE1OywhCpwkAEIIYZVmSWPU8lF8vftrXn7oZSZ2nYiNMvNuaSEKjyQAQggBpKSlELIkhO8Pfs+b7d/kvc7voZQyOywhCo0kAEKIMi8xNZH+i/qz/K/lfBLwCa8//LrZIQlR6CQBEEKUaQnJCfT5rg9hJ8P4svuXPN/qebNDEqJISAIghCizrt68SvcF3dl+bjtz+s5hcNPBZockRJGRBEAIUSZdun6JoLlBHIo+xKL+i+jr29fskIQoUpIACCHKnLNxZwmYE8Dpa6dZ9uQyutbvanZIQhQ5SQCEEGXK8Zjj+M/x52riVdaErOERz0fMDkkIU0gCIIQoMw5eOkjg3ECS05JZP2Q9Laq3MDskIUwjI1wIIcqEned30nF2RzSajcM2SuUvyjxJAIQQpd7m05vpMqcLlcpVInx4OI2rNTY7JCFMJwmAEKJUW3N8DUFzg/Co5MHmpzdT37W+2SEJUSxIAiCEKLWWHF5Cr4W98Kniw6bhm6jpXNPskIQoNiQBEEKUSvP2zaP/ov4092hO2NAwqlWsZnZIQhQrkgAIIUqdqTumMmTJEDp6dWRtyFoqO1Y2OyQhih1JAIQQpcr438fz/Irn6eHTgxXBK6hUrpLZIQlRLEkCIIQoFbTW/Hf9f3l93esMbDyQnwb8RHm78maHJUSxJQMBCSFKPK01Y1aP4Ys/v2BEsxFM6zkNWxtbs8MSoliTBEAIUaKlWdJ4bvlzfLP7G1556BUmdp2IUsrssIQo9grtEoBSaqZS6pJS6kCGZa5KqbVKqaPWf6VnjhAi35LTkgn+KZhvdn/Dfzv8Vyp/IfKgMPsAzAa6ZVr2BvCb1tob+M36XAgh8uxmyk0e//5xfjj4A58GfMp7nd+Tyl+IPCi0BEBrvQmIybS4D/Ct9e9vgccKq3whROmVkJxAjwU9WHl0JVN7TOW1h18zOyQhSpyi7gPgrrWOsv59AXDPbkOl1EhgJICnp2cRhCaEKAmu3rxK9wXd2X5uO3P6zmFw08FmhyREiWTabYBaaw3oHNZP11q31Fq3dHNzK8LIhBDF1cWEi3T6thO7onaxqP8iqfyFuAdF3QJwUSnlobWOUkp5AJeKuHwhRAl15toZAuYGcObaGZY9uYygekFmhyREiVbULQBLgaHWv4cCvxRx+UKIEuhYzDHaz2rPhYQLrAlZI5W/EAWg0FoAlFILgU5AVaXUWeBt4GPgB6XUCOAUMKCwyhdClA4HLh0gcG4gKWkprB+ynhbVW5gdkhClQqElAFrrJ7NZ5V9YZQohSpcd53fQdV5XHGwd2DR8E43cGpkdkhClhswFIIQolsJPhdPl2y44Oziz+enNUvkLUcAkARBCFDurj62m67yuVHeqTvjwcOpWrmt2SEKUOpIACCGKlZ8O/0Svhb3wqeLDpuGbqOlc0+yQhCiVJAEQQhQbc/fOZcCiAbSs3pKwoWFUq1jN7JCEKLUkARBCFAtfbf+KIT8PoaNXR9aErKGyo8wVJkRhkgRACGG6T3//lBdWvkAvn16sCF5BpXKVzA5JiFJPEgAhhGm01ry5/k3+ue6fDGw8kB8H/Eh5u/JmhyVEmVDUQwELIQQAFm1hzKoxTNo2iWeaPcPUnlOxtbE1OywhygxJAIQQRS7NksbIZSOZuWcmY9qMITQoFKWU2WEJUaZIAiCEKFLJackM/mkwiw4t4q0Ob/FOp3ek8hfCBJIACCGKzM2Um/Rf1J8VR1cwPnA8r7Z71eyQhCizJAEQQhSJ+KR4en/Xm42RG5nWcxojW4w0OyQhyjRJAIQQhS7mZgyPzn+Uned3MrfvXJ5q+pTZIQlR5kkCIIQoVBcTLhI0L4iIyxEsHrCYxxo+ZnZIQggkARBCFKIz184QMDeAs3FnWf7kcgLrBZodkhDCShIAIUShOBZzDP85/sQmxrJm8Boe9nzY7JCEEBlIAiCEKHAHLh0gcG4gKWkphA0No7lHc7NDEkJkIkMBCyEK1I7zO+g4uyM2yoZNwzdJ5S9EMSUJgBCiwGw6tYku33bB2cGZ8OHhNHJrZHZIQohsSAIghCgQq46totu8btRwrsHm4ZupW7mu2SEJIXJgSgKglBqjlDqolDqglFqolJLpv4QowX489CO9F/amQdUGbBy2kRrONcwOSQhxF3dNAJRSbZVSU5RS+5RS0Uqp00qplUqpfyilXPJaoFKqBjAaaKm19gNsgUF5D10IURzM2TuHAYsH0LJ6S8KGhlGtYjWzQxJC5EKOCYBS6lfgGWA10A3wABoBbwLlgV+UUr3zUa4d4KiUsgMqAOfzsQ8hhMm+3P4lQ38eSmevzqwJWcN95e8zOyQhRC7d7TbAEK315UzLEoBd1keoUqpqXgrUWp9TSk0ATgM3gTVa6zWZt1NKjQRGAnh6eualCCFEEfhk8ye88dsb9PLpxQ/9f6C8nVzJE6IkybEFIL3yV0pVVErZWP/2UUr1VkrZZ9wmt5RSlYE+QB2gOlBRKTU4i7Kna61baq1burm55aUIIUQh0lrzn9/+wxu/vcGTfk/y44AfpfIXogTKbSfATUB56/X7NUAIMDufZQYAJ7XW0VrrFOAnoF0+9yWEKEIWbeHlVS/z4eYPebb5s8ztOxd7W3uzwxJC5ENuEwCltb4BPA58qbXuDzTOZ5mngTZKqQpKKQX4A4fzuS8hRBFJs6QxYukIJm+bzNg2Y5nWcxq2NrZmhyWEyKdcJwBKqbbAU8AK67J8/c/XWv8JLMboQ7DfGsP0/OxLCFE0ktOSefLHJ5m9ZzZvd3ybCUETMPJ3IURJldu5AF4G/gUs0VofVErVBcLyW6jW+m3g7fy+XghRdG6m3OSJRU+w8uhKJgROYFy7cWaHJIQoALlKALTWmzD6AaQ/P4FxL78QohSLT4qn18JebDq1iWk9pzGyxUizQxJCFJC7jQMwQynVJJt1FZVSTyulniqc0IQQZoq5GUPA3AA2n97MvMfnSeUvRClztxaAKcB/rUnAASAaYwAgb8AZmAnML9QIhRBF7mLCRQLnBnLkyhF+HPAjfRr2MTskIUQByzEB0FrvAQYopSoBLTFGArwJHNZaHymC+IQQRez0tdMEzAngXPw5VgSvIKBugNkhCSEKQW77ACQAGwo3FCGE2Y5eOUrA3ACuJV5jbcha2tWSITqEKK1yexeAEKKU239xP4FzA0nTaawfup7mHs3NDkkIUYhMmQ5YCFG8bD+3nU7fdsLWxpZNwzZJ5S9EGZCnBEApVaGwAhFCmGPTqU34z/HHxcGF8OHh+Lr5mh2SEKII5CoBUEq1U0odAiKszx9QSn1ZqJEJIQrdqmOr6DqvKzWcaxA+PJy6leuaHZIQoojktgXgM6ArcAVAa70X6FBYQQkhCt+Ph36k98Le+Fb1ZdOwTdRwrmF2SEKIIpTrSwBa6zOZFqUVcCxCiCLy7Z5vGbB4AK1qtGL90PW4VZQpt4Uoa3KbAJxRSrUDtFLKXin1KjKDnxAl0pRtUxj2yzA6e3VmzeA13Ff+PrNDEkKYILcJwCjgH0AN4BzwoPW5EKIE+Xjzx7z464v0btCb5cHLqViuotkhCSFMktuBgC5jTAUshCiBtNb8Z/1/+GjzRwQ3CWZ2n9nY29qbHZYQwkS5SgCUUnWAlwCvjK/RWvcunLCEEAXFoi28/OvL/G/7/xjZfCRf9vgSWxtbs8MSQpgstyMB/gx8AywDLIUXjhCiIKVaUnl22bPM3jObcW3HMT5wPEops8MSQhQDuU0AErXWkwo1EiFEgUpOS+apn55i8aHFvNPxHd7q+JZU/kKIW3KbAHyhlHobWAMkpS/UWu8qlKiEEPfkRsoNnvjhCX499iuhQaGMbTvW7JCEEMVMbhOAJkAI0IXblwC09bkQohiJS4qj18JehJ8KZ3rP6Tzb4lmzQxJCFEO5TQD6A3W11smFGYwQ4t7E3Iyh27xu7IraxfzH5/NkkyfNDkkIUUzldhyAA0CBjRailLpPKbVYKRWhlDqslGpbUPsWoqy6kHCBjrM7su/iPn4a+JNU/kKIHOW2BeA+IEIptZ07+wDk9zbAL4BVWusnlFLlAJllUIh7cPraaQLmBHAu/hwrglfgX9ff7JCEEMVcbhOAtwuqQKWUC8ZEQsMArJcV5NKCEPl09MpR/Of4E5cUx9qQtbSr1c7skIQQJUBuRwLcWIBl1gGigVlKqQeAncDLWuvrGTdSSo0ERgJ4enoWYPFClB77L+4ncG4gaTqNsKFhNPNoZnZIQogSIsc+AEqpzdZ/45VScRke8UqpuHyWaQc0B77SWjcDrgNvZN5Iaz1da91Sa93SzU1mKhMis23nttFxdkfsbOzYNGyTVP5CiDy5WyfAigBaayettXOGh5PW2jmfZZ4Fzmqt/7Q+X4yREAghcmlj5Eb85/hzX/n7CB8ejq+br9khCSFKmLslALqgC9RaX8CYXriBdZE/cKigyxGitPr16K90m9+NWs61CB8eTp3KdcwOSQhRAt2tD0A1pVS2Q4hprSfms9yXgPnWOwBOAMPzuR8hypTFhxYT/GMwftX8WD14NW4V5fKYECJ/7pYA2AKVgAIdQFxrvQdoWZD7FKK0m71nNiOWjqBtzbasCF6BS3kXs0MSQpRgd0sAorTW7xVJJEKIbP1v2/946deXCKgbwM8Df6ZiuYpmhySEKOHu1gdApg4TwmQfhX/ES7++RJ8GfVj25DKp/IUQBeJuLQAynJgQJtFa8+/f/s3Hv39McJNgZveZjb2tvdlhCSFKiRwTAK11TFEFIoS4zaItjP51NFO2T+G5Fs/xZY8vsVG5nbpDCCHuLrdDAQshikiqJZVnlj7Dt3u/ZVzbcYwPHI9ScjVOCFGwJAEQohhJTksm+Mdgfjz8I+91eo83O7wplb8o0Xbu3FnNzs7ua8CP3M9AKwqGBTiQmpr6TIsWLS5lXikJgBDFxI2UG/T7oR+rjq3is66f8UqbV8wOSYh7Zmdn9/X999/v6+bmdtXGxqbAB5cT2bNYLCo6OrrRhQsXvgb+NnuvZGNCFANxSXE8Ov9RVh9bzYxeM6TyF6WJn5ubW5xU/kXPxsZGu7m5XcNoffkbaQEQwmRXblzh0fmPsvvCbhb0W8Agv0FmhyREQbKRyt881vc+y5N9SQCEMNGFhAsEzg3k6JWj/DTgJ3o16GV2SEKIMkIuAQhhklOxp2g/qz0nr55kRfAKqfyFKARHjhwp5+3t3djsOLLTu3fvOhNP5UYAACAASURBVF5eXn7e3t6N+/fv75WUlFRkvX4lARDCBH9d+Yv2s9oTfT2atSFr8a8rY24JURY99dRTMSdOnDhw5MiRg4mJierzzz+vWlRlSwIgRBHbd3EfHWZ1IDE1kQ3DNtC2VluzQxKiTDh06FA5X1/fRhs3bqyQmprKc889V9PPz8/Xx8en0fjx46sCWCwWnnvuuZre3t6NfXx8Gs2YMaMywPLly51atmzZoFOnTvW9vLz8goODPdPS0khNTaVfv35e6du/++671fIS08CBA6/Z2NhgY2NDy5Ytr589e7ZcYRx7VqQPgBBF6M+zf/Lo/EepYF+BdUPW0bBqQ7NDEqLIPP00tQ4coEJB7tPPjxszZ3Lmbtvt3bvXYdCgQfVmzpx5sm3btjcnTJhQ1cXFJe3AgQOHb968qVq1atWwV69ecVu3bq2wf/9+x8OHDx+Mioqya926tW9QUFACwP79+yvu3r37gI+PT3KHDh2858yZU7l+/fpJUVFR9kePHj0IcPnyZdv8HEdSUpL6/vvvq0ycOPGux1JQpAVAiCKyIXIDAXMDqOxYmfDh4VL5C1FEYmJi7B577LH68+bNO9G2bdubAOvWrXP+4YcfqjRs2LBRs2bNfK9evWp36NCh8uHh4U4DBgyIsbOzo1atWqkPPfRQwubNmysANGnS5HqjRo2S7ezsGDBgQEx4eHilhg0bJp05c8Zh6NChtRYvXuxcuXLltPzEOHToUM82bdokdOvWLaEgjz0n0gIgRBFYeXQl/X7oR93KdVkbspbqTtXNDkmIIpebM/XC4OTklFa9evXksLCwSi1atEgE0Fqr0NDQ0/369YvLuO2KFStcsttP5lE5lVK4ubmlHThw4NCSJUucp06d6vb999+7Llq0KDJ9m9TUVPz8/BoBdOvWLfbzzz8/n3m/48aN87h8+bLd6tWrj9/bkeaNtAAIUcgWHVzEY989RiO3RmwctlEqfyGKmL29vf7111+PL1y4sMrUqVNdAQIDA6999dVXbum97vft2+cQFxdn06FDh/jFixe7pqamcv78ebtt27ZVat++/XUwLgFERESUS0tLY/Hixa7t27ePj4qKsktLS2PYsGGxH3300bn9+/ffcYnDzs6OiIiIQxEREYeyqvwnTpxYdf369S4///zzCVvbfF09yDdpARCiEM3aPYtnlj1D25ptWRG8Apfy2Z5cCCEKkbOzs2X16tXHOnXq5OPk5JQ2ZsyYy5GRkQ5NmjTx1VorV1fXlJUrVx4PCQmJ3bJlSyVfX9/GSin97rvvnvX09Ezdt28ffn5+10eNGuUZGRlZvl27dnEhISGx27ZtcxwxYoSXxWJRAO+9997ZvMT1+uuv1/bw8Ehq2bKlL0DPnj2vTpgwIaow3oPMlNbFf4Cmli1b6h07dpgdhhB5MvnPyYxeNZrAuoEsGbiEiuUqmh2SKGOUUju11i3NjGHv3r2RDzzwwGUzYygIy5cvdwoNDXUPCws7ZnYsebV3796qDzzwgFfm5aZdAlBK2SqldiullpsVgxCF5cPwDxm9ajSPNXyMZU8uk8pf5NnNm7B6NcTHmx2JKK3M7APwMnDYxPKFKHBaa95Y9wb/Wf8fBjcdzKL+i3CwczA7LFECaA1//QWTJkH37lClCnTrBr/9ZnZkAqBnz57xJfHsPyem9AFQStUEegD/B4w1IwYhCppFW3hp5Ut8ueNLRrUYxZQeU7BR0s9WZC8hAcLCYNUq+PVXOHnSWN6gAYwcaSQAHTuaG6MovczqBPg58DrgZFL5QhSoVEsqI5aOYM7eObza9lU+Dfz0b7cMCaE1HDx4u8IPD4eUFKhYEfz94bXXjEq/Th2zIxVlQZEnAEqpnsAlrfVOpVSnHLYbCYwE8PT0LKLohMi7pNQkgn8K5qfDP/F+5/f5T/v/SOUvbomNhXXrjEp/1So4d85Y3qQJvPIKPPooPPwwlCuyAWCFMJjRAvAw0Fsp1R0oDzgrpeZprQdn3EhrPR2YDsZdAEUfphB3dyPlBo9//zirj6/ms66f8UqbV8wOSZjMYoHdu29X+H/8AWlp4OICgYFGhd+1K9SoYXakoqwr8guUWut/aa1raq29gEHA+syVvxAlQVxSHN3mdWPN8TV83etrqfzLsMuXYcECGDIEPDygZUt4802jJ/+//gWbNxvbLFoETz8tlX9x98Ybb9xfUPsaO3Zs9bfeess9v6/fsmWL44MPPtiwfv36d0xOVBBkICAh8uHKjSt0m9+NPRf2sLDfQgb6DTQ7JFGE0tJg27bbZ/nbtxvX96tWNc7uu3WDoCColqd54URxMWnSJI+PP/74Qm63t1gsaK0pjJH8KlWqZJk7d+7JJk2aJEVGRtq3atXKt2/fvnFVq1bN15wDGZnaRVlrvUFr3dPMGITIq6j4KDrO7sj+i/tZMnCJVP5lRFQUzJ4NgwaBmxu0awcffAB2dvDuu0ZCcOECzJsHgwdL5V9cHDlypFydOnUa9+7du07dunUbd+vWrW58fLzN0qVLnQICAuqlb7dkyRLnwMDAei+88EKNpKQkm4YNGzbq3bt3HYB33nnH3dvbu7G3t3fj9957r1r6fr28vPz69u3r5ePj0/j48ePlFi9e7NyoUSPfBg0aNGrbtq1P+r4PHz7s2Lp16wY1a9Zs8sEHH+Tpm9G0adOkJk2aJAF4eXmluLq6pkZFRRXIybu0AAiRB/sv7qfv9325kHCBlU+tpEudLmaHJApJSgps2XK7x/7evcZyDw947DHjLD8gAFxdzY2zpGndmgaZlz3+ODFvvEF0fDw2/v54Z14/eDCXR4/mSlQUdn36UC/jum3bOHK3MiMjI8tPmzYtMigo6Hr//v29xo8f7/bOO+9cfPnllz3Pnz9vV7169dSZM2dWGT58+OXg4OBrs2fPrhYREXEIIDw8vMKCBQuq7Ny587DWmhYtWvj6+/vHV61aNe306dMO33zzzUl/f//I8+fP27344oteGzZsiGjYsGHyxYsXbzUHHDt2rPyWLVuOxMbG2vr6+vq99tpr0Q4ODnnu2xYWFlYhJSVFNWrUKCmvr82K3KQsxF1orVl3Yh2Pzn+UplObcuXmFdaGrJXKvxQ6fRqmT4fHHzcG4unUCSZMgMqV4eOPYc8eoxf/zJkwYIBU/iXF/fffnxwUFHQdICQk5MqWLVsq2djYMGDAgCszZsxwvXz5su2uXbsq9e/f/1rm127YsKFS9+7dY52dnS0uLi6WHj16XA0LC3MC8PDwSPb3979u3a5i69at4xs2bJgM4O7ufquJPigoKNbR0VF7eHikurq6ppw9ezbPJ9+nTp2yHz58eN0ZM2ZEFtSlBmkBECIbyWnJLNy/kIlbJ7Lv4j7cK7rzfuf3GdVyFFUrVDU7PFEAEhONe/HTz/IPW8cm9fSE4GDjLL9LF3B2NjfO0iSnM3YnJyw5rffwIDU3Z/yZZTWNL8Dzzz9/pUePHvXLly+ve/XqddXe3j5P+61QoYIlN9tlPNu3tbUlNTX1joDmzJlz34cfflgdYPr06ZEdOnS4kXF9TEyMzaOPPlr/7bffPpeecBQESQCEyCTmZgzTdkxj8rbJRCVE0ditMTN7zyS4SbAM61sKHDt2u8IPCzN66js4GCPuPfusUek3bAgylEPpERUVVW7dunUVAwICrs+fP9+1Xbt2CWBcU3d3d08JDQ31WLVq1V/p29vZ2emkpCTl4OCgO3funPD00097vf/++xe01qxcubLy7NmzT2Quo1OnTtfHjh1bOyIiolz6JYCMrQA5GTJkSOyQIUNis1qXmJioevToUX/QoEFXhg8ffjW/70FWJAEQwup4zHE+3/o5M/fM5EbKDQLrBjKrzyyC6gXJwD4l2PXrsGGDUeGvWgXHjxvLvb3hmWeMCr9TJ6hQIae9iJLMy8srcfLkydVGjhxZwdvbO/HVV1+NTl83aNCgK1OmTLFr3rx5Yvqyp556KtrX17eRn5/fjaVLl54MDg6+0rx5c1+AkJCQ6IcffvjmkSNH7hi6qXr16qmTJk2K7Nu3b32LxUKVKlVStmzZcvReY585c2bl7du3V7p69ardggULqlqXnWzXrt3Ne923TAcsyrwtZ7YQ+kcoSw4vwc7GjuAmwYxtO5am7k3NDk3kg9ZGU356hb9pEyQnGxV8ly5Ghd+tG9Srd/d9lXQyHbDRW79nz57eR48ePZjV+iFDhng2a9bsxpgxY0r8lMXZyW46YGkBEGVSmiWNJRFLCP0jlK1nt1K5fGXeeOQNXmz9ItWdqpsdnsija9eMWfPS78s/c8ZY3rgxvPSSMfreI48YTf1CpGvcuLGvo6OjZdq0aWfMjsUMkgDco8fen8Er3XvQqYVUGiVBfFI8s/bM4vOtn3My9iR1K9dl8qOTGf7gcCqWq2h2eCKXLBbjtrz0Cn/LFkhNNTrrBQTAW28ZA/LUqmV2pMJsDRo0SM7u7P/gwYNlekp6SQDuwaxfd/OLZSRp6z6kU4t/mR2OyMG5uHNM+nMS03ZO41rSNdrVaseEoAn0adAHW5uCH71LFLwrV2Dt2tuV/sWLxvJmzeD1141m/TZtII8duYUosyQBuAdRcZepcK0ZU1983uxQRDb2XNhD6B+hfHfgOyzawuO+jzOu7Tja1GxjdmjiLtLSYMeO2z32t20zru+7ut453O79BTZquxBliyQA9+DfAwP598BAs8MQmVi0hVXHVhH6RyjrT66non1FXmj5Aq+0eYU6lWWi9eLswgVYs8ao8NesgZgY43a8hx6Ct982Kv2WLaEQhlwXosyRBCCfJv64iaEBraji4mh2KMIqMTWRefvmMfGPiRy+fJgaTjX4JOATRrYYyX3l7zM7PJGFlBTYuvV2j/3du43l7u7Qq5dR4QcGGqPyCSEKlgwFnA+7/rrIuD1BdB0v1/2Lg8s3LvPexveo/Xltnl32LA52DsztO5cTL5/g9Ydfl8q/mDlzBmbMgH79jNnzOnSATz8FJyf48EMjCTh//vbEO1L5i3tx5MiRct7e3o3NjiM7H374oZunp6efUqpFxkl+LBYLw4YNq+Xp6enn4+PTaPPmzQU+UoW0AOTDqFlfgEMy4x9/wexQyrQjl4/w2dbP+HbvtySmJtLduzvj2o6js1dnGbinGElKuj3c7qpVcNDaH7tWLRg40LhFr0sXcHExN04hzNCxY8eEfv36XevSpcsdkyQtWrTI5cSJE+UjIyMPhIWFVXzhhRc89+3bF1GQZUsLQB5FXohjO1/ieb0fnZv63P0FokBprdkYuZHeC3vTcEpDZu+ZzeAmgzn4wkFWBK+gS50uUvkXAydOwJQpRjO+q6vRjD95sjGT3oQJRhJw6pQx8U7fvlL5i6Jx6NChcr6+vo02btxYITU1leeee66mn5+fr4+PT6Px48dXBePM+7nnnqvp7e3d2MfHp9GMGTMqAyxfvtypZcuWDTp16lTfy8vLLzg42DMtLY3U1FT69evnlb79u+++m6fpfh9++OGbDRo0SM68/JdffrnvqaeeumJjY4O/v//1uLg4u1OnThXoPS7SApBHz02fBuWv8WmXf5odSpmSkpbC4kOLCf0jlJ1RO6laoSpvdXiLF1q9gHsld7PDK/Nu3DCG200/yz9qHQC1bl0YPtw4y+/UCSrKUAtl2tO/PF3rwKUDBdqU7VfN78bMPjPvOpDP3r17HQYNGlRv5syZJ9u2bXtzwoQJVV1cXNIOHDhw+ObNm6pVq1YNe/XqFbd169YK+/fvdzx8+PDBqKgou9atW/sGBQUlAOzfv7/i7t27D/j4+CR36NDBe86cOZXr16+fFBUVZZ8+1sDly5cLpItqVFSUvZeX163EwMPDI/nUqVP2tWvXTimI/YMkAHm2L2YLVWz8Gdje1NE1y4xrideYsWsGk/6cxJm4MzSo0oCpPaYy5IEhONpLB0yzaA0REbcr/I0bjaZ+R0fo3BlGjzY68NWvb3akQkBMTIzdY489Vn/x4sXHW7RokQiwbt0654iIiApLly6tDBAfH2976NCh8uHh4U4DBgyIsbOzo1atWqkPPfRQwubNmyu4uLhYmjRpcr1Ro0bJAAMGDIgJDw+v1LNnz7gzZ844DB06tFavXr2u9e3bN87MY80LSQDy6NzEn4i6WmI+3xLrVOwpvvjzC77e9TXxyfF08urElz2+pLt3d2yUXLkyQ1wcrF9/u9I/dcpY7usL//iHUeG3bw/ly5sbpyi+cnOmXhicnJzSqlevnhwWFlYpPQHQWqvQ0NDT/fr1u+MHfcWKFdlekMpqWmE3N7e0AwcOHFqyZInz1KlT3b7//nvXRYsWRaZvk5qaip+fXyOAbt26xX7++efncxOzh4dHSmRk5K0Jh6KiosoV5Nk/SB+AXEtKTiPi9GVsbBQ1qsgFy8Ky/dx2Bi0eRL1J9Zj05yR6NejFjmd3EDY0jJ4+PaXyL0JaG8PtfvKJ0XxfpYpxvX7BAmjeHKZNg8hIOHQIQkON6/xS+YviyN7eXv/666/HFy5cWGXq1KmuAIGBgde++uort6SkJAWwb98+h7i4OJsOHTrEL1682DU1NZXz58/bbdu2rVL79u2vg3EJICIiolxaWhqLFy92bd++fXxUVJRdWloaw4YNi/3oo4/O7d+//45LHHZ2dkRERByKiIg4lNvKH6B3796x8+fPr2KxWPjtt98qOjk5pRV0AiAtALn0+qyfmXQmhEXdtvDEIw+aHU6pYtEWlh1ZRugfoYSfDsfZwZkxbcYw+qHR1HKRwdyLUkwMrFtn3Je/ejVERRnLH3wQXn3VOMtv2xbKlct5P0IUN87OzpbVq1cf69Spk4+Tk1PamDFjLkdGRjo0adLEV2utXF1dU1auXHk8JCQkdsuWLZV8fX0bK6X0u+++e9bT0zN13759+Pn5XR81apRnZGRk+Xbt2sWFhITEbtu2zXHEiBFeFotFAbz33ntn8xLXBx98UG3y5Mn3X7lyxf6BBx5o1Llz52vff//9qQEDBlxbsWKFS+3atf0cHR0tX3/9dWRBvydFPh2wUqoWMAdwBzQwXWv9RU6vMXs6YItFU2nMQ6TZXyXh4wjs7WQYsoJwI+UG3+75ls+2fsbRmKN4unjyykOvMKL5CJwdnM0Or0ywWGDnztsD8fz5p7GscmVjmN1u3Yxhdz08zI5U5IdMB1xwli9f7hQaGuoeFhZ2zOxY8qo4TQecCozTWu9SSjkBO5VSa7XWh0yIJVc+XbSBm67bGVplqlT+BeBCwgWmbJvCVzu+4srNK7Sq3orv+n1Hv0b9sLORRqnCdumScXa/apUx3O7ly8Zwu61awZtvGpV+69Yy3K4QpV2R/9pqraOAKOvf8Uqpw0ANoPgmAL9/jI2jO5NeG2p2KCXawUsHmfjHRObtn0dKWgq9G/RmXNtxPOL5iNy7X4hSU43hdtM77+3caSyvVs24PS99Up2qVc2NU4jirGfPnvE9e/aMNzuOgmTq6ZZSygtoBvyZxbqRwEgAT0/PIo0ro992neSq61p6lP8Q5wrSwymvtNb8dvI3Qv8IZdWxVTjaOTKi2QjGtBmDdxVvs8Mrtc6evX2Wv3YtXLtmnNG3bQsffGBU/A8+CDbSp1KIMsu0BEApVQn4EXhFa/23++q01tOB6WD0ASji8G7xb16HpckHaO5dw6wQSqTktGQW7l/IxK0T2XdxH+4V3fmg8weMajmKKhVkcPeClpQEv/9++yx//35jeY0a8MQTRoXv7w/3ybQIQggrUxIApZQ9RuU/X2v9kxkx5EZamsbWVtGrTSOzQykxYm7GMG3HNCZvm0xUQhSN3Rozs/dMgpsE42DnYHZ4pcrJk7cr/N9+g+vXwd7euBd//Hijab9xY+P6vhBCZFbkCYAyLvZ+AxzWWk8s6vLzoukbL3HTco1j4+dgYyO/ojk5HnOcz7d+zsw9M7mRcoPAuoHM6jOLoHpBcn2/gNy8aYy4t2qV0Wv/r7+M5V5eMGSIcZbfuTNUqmRqmEKIEsKMK4APAyFAF6XUHuujuwlx5Gj30YscKv81jvaOUvnnYMuZLfT7oR/ek72ZtnMa/Rv1Z++ovawJWUPX+l2l8r8HWsORI/DFF8bZvKurUclPm2aMsf/FF8b6Eyfgyy+NiXek8hfi3r3xxhv3F9S+xo4dW/2tt966pwlL2rdv7+3k5PRg586d7xhcOyIiolzTpk0benp6+vXo0aNuYmJinn5wizwB0Fpv1lorrXVTrfWD1sfKoo7jbp6fNQlsk5n05Ktmh1LspFnSWHxoMW2/acvDMx8m7GQYbzzyBpGvRDL7sdk0dW9qdoglVnw8LF0Kzz9vVPING8Irrxgj7o0aZZz9x8QYLQCjR4OPjzTxC1HQJk2alKeRLywWC2lpaYUVDq+++uqFadOmncy8fOzYsTVffPHFi6dPnz7g4uKS+sUXX+TpXh7pA5yFUxfi+FNPoVZCP/wfkCl/08UnxTPpz0l4T/am/6L+XLp+icmPTubMmDN86P8h1Z2qmx1iiaO10WHv00+hSxdjuN0+fWDePHjgAfjqK+MMPyICPvvMGJTHUeZAEiLXjhw5Uq5OnTqNe/fuXadu3bqNu3XrVjc+Pt5m6dKlTgEBAfXSt1uyZIlzYGBgvRdeeKFGUlKSTcOGDRv17t27DsA777zj7u3t3djb27vxe++9Vy19v15eXn59+/b18vHxaXz8+PFyixcvdm7UqJFvgwYNGrVt2/ZW5XH48GHH1q1bN6hZs2aTDz74IE/TBQP06dMn3tnZ2ZJxmcVi4Y8//nAaPnz4VYCnn376yrJly/LUzVdGXcnCqBnTofw1Pu4qU/4CnIs7x6Q/JzFt5zSuJV2jXa12TAiaQJ8GfbC1kdFi8io21rg1L70D33nr6OBNm8KYMUZz/8MPy3C7onRqPaN1g8zLHvd9POaNR96Ijk+Kt/Gf4/+3+4MHNx18efRDo69ExUfZ9fmuT72M67Y9u+3I3cqMjIwsP23atMigoKDr/fv39xo/frzbO++8c/Hll1/2PH/+vF316tVTZ86cWWX48OGXg4ODr82ePbtaRETEIYDw8PAKCxYsqLJz587DWmtatGjh6+/vH1+1atW006dPO3zzzTcn/f39I8+fP2/34osvem3YsCGiYcOGyRcvXrz143js2LHyW7ZsORIbG2vr6+vr99prr0U7ODjc091tFy9etHNyckqzt7cHwMvLK/nixYt5+tWQBCALnw4eTLXlzgR3KttT/u65sIfQP0L57sB3WLSFx30fZ1zbcbSp2cbs0EoUiwV27bpd4W/dCmlpxi15gYG3h9utIXeaClEo7r///uSgoKDrACEhIVcmTZpUzcbG5uKAAQOuzJgxw/Uf//jHlV27dlX66aef/tbMvmHDhkrdu3ePTT8D79Gjx9WwsDCn/v37x3p4eCT7+/tft25XsXXr1vENGzZMBnB3d791TSAoKCjW0dFROzo6prq6uqacPXvWrl69egU6sU9+SAKQhSZ17ufbl0aaHYYpLNrCqmOrCP0jlPUn11PRviL/aPUPXn7oZepUrmN2eCVGdLQxzO6qVcaAPNHRxvKWLeFf/zIq/YceAjv5HyjKmJzO2J0cnCw5rfdw8kjNzRl/ZllN4wvw/PPPX+nRo0f98uXL6169el1NP5vOrQoVKljuvhVkPNu3tbUlNTX1joDmzJlz34cfflgdYPr06ZEdOnS4cbd9uru7p8bHx9umpKRgb29PZGRkOXd39+S8xC99ADJISk6jztgh/G9ZuNmhFLnE1ES+3vU1fl/60WNBD45cPsInAZ9wduxZPu/2uVT+d5GaClu2wFtvGePou7vD4MFGAhAUBHPnwsWLsH07vP++0cQvlb8QRSMqKqrcunXrKgLMnz/ftV27dgkAXl5eKe7u7imhoaEeI0eOvDVhkZ2dnU6fJrhz584JK1euvC8+Pt4mLi7OZuXKlZU7d+78tyGBO3XqdH3btm1OERER5QAyXgK4myFDhsSmTxmcm8ofwMbGhjZt2sTPmjWrMsDMmTOr9OzZMza3ZYK0ANzh33N+IdJlLpGXe5odSpGJvh7NVzu+Ysr2KVy6fokH73+QuX3nMqDxAMrZykXonJw/f+dwu1evGkPrtmkD775r3LLXvLkMtyuE2by8vBInT55cbeTIkRW8vb0TX3311ej0dYMGDboyZcoUu+bNmyemL3vqqaeifX19G/n5+d1YunTpyeDg4CvNmzf3BQgJCYl++OGHbx45cuSOH8jq1aunTpo0KbJv3771LRYLVapUSdmyZcvRgoi/RYsWDU6cOFH+5s2btu7u7k2//PLLyH79+sWFhoaeHThwYL0PPvigRuPGjW+8/PLLeZp1scinA86PopgO2GLROI19iBS7GK5/fKTUz/p35PIRPtv6Gd/u/ZbE1ES6e3dnXNtxdPbqLPfuZyE2Fo4fNx67dhm34e3bZ6zz8DCa9B99FAICjKl0hSgOZDpgo7d+z549vY8ePXowq/VDhgzxbNas2Y0xY8aU+CmLs1OcpgMulkJ/2sCNytsJua/0TvmrtWbTqU2E/hHKsr+W4WDrQEjTEMa0HUMjt7I93LHFYpzRHz9u3HaXXtmnP2Jibm9rbw+PPAKffGJU/E2ayL34QpREjRs39nV0dLRMmzbtjNmxmEESAKuPwz/Gprw7k18tfVP+pqSlsPjQYkL/CGVn1E6qVqjKWx3e4oVWL+Be6Z4GqCpRkpKMAXUyV+7Hjxvj6icm3t7W1hY8PaFePejf3/g3/VG/PlSsaNphCCHyoEGDBsnZnf0fPHjwcFHHU5xIAoDR/N+pZnecyj+BS8XSM+XvtcRrzNg1g0l/TuJM3BkaVGnA1B5TGfLAEBztS+doMteuZV3BHz8OZ84YA++kq1DBqNAbNDCa7zNW8rVrG2f6Qoh7ZrFYLMrGxqb4X28uhSwWiwKyvFtBEgDAxkbx42svmx1GgTkVzGg0aQAADaxJREFUe4ov/vyCr3d9TXxyPJ28OvFljy/p7t0dG1Wye6RpDVFR2VfyV67cub2bm1Ght29/ZwVfr57RU1+a7oUodAeio6Mbubm5XZMkoGhZLBYVHR3tAhzIan2ZTwDmrdvN2gO7mPzsYJwrluzparef207oH6EsPrQYgIF+AxnbZiwtqrcwObK8SU6GU6eyruBPnDBmxUtnY3O7qb5fvzsr+Lp1wdnZvOMQQkBqauozFy5c+PrChQt+yK3nRc0CHEhNTX0mq5VlOgH4fsM+hqwLxMbiyOuXe9K4Ysm7Hm7RFpYdWUboH6GEnw7H2cGZMW3GMPqh0dRyqWV2eNmKj8/+LP70aaNTXjpHR6Myr1fPuKc+c1O9DJkrRPHVokWLS0Bvs+MQf1dmE4DvwvYSvNofG4sj6waH0bh2yan8LyRcYOf5new4v4P5++dzNOYoni6eTAyayIjmI3B2MP+0V2tj4JvsKvno6Du3r1LFqNDbtjUG0MlYyXt4SFO9EEIUtDKZAGSs/H8bvIGOTevd/UUmuZhwkZ1RRmW/M2onO8/v5Fz8OQAUiodqPsT7nd+nX6N+2NkU7ceZkmKcrWfXVH/9+u1tlYJatYwKvU+fv1+Pd3Ep0tCFEKLMK5MJQNjh/dikVeS3kPXFqvJPr+x3nt95q9LPWNk3qNqATl6daFm9JS08WvDg/Q/i5OBUqDElJGR9X/zx48Z1+oxTYDs43G6q79Llzgrey8tYL4QQongoUyMBxiYkcl8l4za/K3E3qOJc4Z73mV+Xrl+6o6LfGbWTs3FnAaOy96nic6uib1G9Bc3ub1Yolb3WRnN8dk31Fy/euX3lyn8/e09/VK8uw94KUZwUh5EARfFVZloAvtuwh6dW9OSTNnN4tV+XIq38o69H3zqz3xG1g53nd3Im7vbAUw2qNKBD7Q608GhBy+otC7yyT029s6k+8xl9QsKd29esaVToPXr8vZKXYW6FEKJ0KBMJwMINu3lqVQA2VKSVt1ehlnX5xuVbHfR2Rhln+Kevnb613qeKD494PnLr7L6ZR7MC6bR3/XrOTfWpqbe3LVcO6tQxKvQOHe6s4OvUgfKlZywkIYQQ2Sj1CcCtyj+tIr+FbKBj07r3vE+tNTE3Yzh97TSnr53mYPTBW2f4p66durWdt6s37Wq1Y3Tr0bea8V3K3723W3KyMfnM1at3/pvV3+nj11+4cOc+XFyMCr15878PZVujhjHUrRBCiLLLlARAKdUN+AKwBb7WWn9cGOVs2ncyX5V/UmoSZ+PO3qrgbz3ibv99I+XOKZvru9anba22vNj6RR50b4F3xeZYbrrcqrBjdsHiHCryjH9nHOgmK/b2RlN85cpQrZoxIU3mpnpXV7l1TgghRPaKPAFQStkCU4BA4CywXSm1VGt9qKDLate4Nm3LPcf7jw+nifd9HI85TszNGK4mXiU6IYbo+KtEX48hOuEKZ+LOcDb+NFHX/7+9e43RorrjOP79lduyLLi71RgKrmKltdQoKrFYW+OFoGBT+sImmBrR1pjW2LTVWm/tiza92EtMNW1qqZeWxqqVGqsGY6hiTAygYFHwvhWrGBFEYUGtXPz3xTmrw7IL7Lo8s/vM75NMnpkzZ2bOec7AnOfM2XNeZsPWtbucq4kDGUMbTds/y8RtM2j4XxtD327jY5vbeH/DYWxZ38zijXDfRujo2HnM+V2/A2huTktLS/o8/PAP14vh3a03NPjhbmZmH00ZLQDHAe0R8SKApNuAWUC/VwBO/umVLNnxR06962rQbp7IWxuh4yDY1AabzsifhaVjPFu2N7CF9Kdso0aliWQ6l6amNBztkUfu3UN89Gj3ljczs3KVUQEYBxTnXl4DfK5rJEkXABcAtLW19elCn2k9irXtZ9OoVhrVQtOQVkYPbWH0sBbGDGuluaGF5hEtNDU00Ni464O96zJypN+dm5lZfRiwnQAjYi4wF9I4AH05x9xvnwWc1Z/JMjMzqwtlNES/ChRnqRmfw8zMzKxGyqgAPAZMlDRB0nBgNnB3CekwMzOrrJq/AoiI7ZIuAu4n/RngTRHxVK3TYWZmVmWl9AGIiAXAgjKubWZmZuW8AjAzM7OSuQJgZmZWQa4AmJmZVZArAGZmZhWk2N2g9QOEpPXAf/cQbX/gjRokZ6BxvqvF+a6ej5L3gyPigP5MjNWPQVEB2BuSlkXElLLTUWvOd7U439VT5bzbvuVXAGZmZhXkCoCZmVkF1VMFYG7ZCSiJ810tznf1VDnvtg/VTR8AMzMz23v11AJgZmZme8kVADMzswoa9BUASadLek5Su6TLy05PX0g6SNIiSU9LekrSd3J4q6SFkl7Iny05XJKuy3l+UtIxhXPNyfFfkDSnEH6spJX5mOskqfY57Z6kIZL+LenevD1B0tKc1tvztNFIGpG32/P+QwrnuCKHPyfptEL4gLw/JDVLmi/pWUnPSDq+CuUt6Xv5Hl8l6VZJDfVa3pJukrRO0qpC2D4v456uYbaLiBi0C2k64f8AhwLDgSeASWWnqw/5GAsck9dHA88Dk4BfAZfn8MuBX+b1mcB9gICpwNIc3gq8mD9b8npL3vdojqt87Iyy813I/8XA34B78/bfgdl5/XrgW3n9QuD6vD4buD2vT8plPwKYkO+JIQP5/gD+Apyf14cDzfVe3sA4YDUwslDO59ZreQMnAscAqwph+7yMe7qGFy9dl8HeAnAc0B4RL0bEVuA2YFbJaeq1iHgtIh7P65uBZ0j/Wc4iPSjIn1/J67OAeZEsAZoljQVOAxZGxJsR8RawEDg97xsTEUsiIoB5hXOVStJ44Azghrwt4BRgfo7SNd+d38d84NQcfxZwW0S8FxGrgXbSvTEg7w9J+5EeDjcCRMTWiNhIBcqbNAX5SElDgUbgNeq0vCPiYeDNLsG1KOOermG2k8FeARgHvFLYXpPDBq3czHk0sBQ4MCJey7vWAgfm9Z7yvbvwNd2EDwS/BX4AvJ+3Pw5sjIjtebuY1g/yl/dvyvF7+32UbQKwHrg5v/q4QdIo6ry8I+JV4DfAy6QH/yZgOfVf3kW1KOOermG2k8FeAagrkpqAfwDfjYiO4r5cy6+rv9mU9CVgXUQsLzstNTaU1DT8h4g4Gnib1FT7gTot7xbSr9MJwCeAUcDppSaqRLUo43q8j6z/DPYKwKvAQYXt8Tls0JE0jPTwvyUi7szBr+emPvLnuhzeU753Fz6+m/CynQB8WdJLpObaU4BrSc2fQ3OcYlo/yF/evx+wgd5/H2VbA6yJiKV5ez6pQlDv5T0NWB0R6yNiG3An6R6o9/IuqkUZ93QNs50M9grAY8DE3It4OKmj0N0lp6nX8nvNG4FnIuKawq67gc5ev3OAfxbCz8k9h6cCm3KT3/3AdEkt+dfWdOD+vK9D0tR8rXMK5ypNRFwREeMj4hBS2T0YEV8DFgFn5mhd8935fZyZ40cOn517jU8AJpI6SA3I+yMi1gKvSPp0DjoVeJo6L29S0/9USY05XZ35ruvy7qIWZdzTNcx2VnYvxI+6kHrPPk/q/XtV2enpYx6+QGqmexJYkZeZpPedDwAvAP8CWnN8Ab/PeV4JTCmc6+ukTlHtwHmF8CnAqnzM78ijQA6UBTiJD/8K4FDSf+jtwB3AiBzekLfb8/5DC8dflfP2HIUe7wP1/gAmA8tymd9F6uFd9+UN/Bh4Nqftr6Se/HVZ3sCtpL4O20itPt+oRRn3dA0vXrouHgrYzMysggb7KwAzMzPrA1cAzMzMKsgVADMzswpyBcDMzKyCXAEwMzOrIFcArK5J2iFphaQnJD0u6fN7iN8s6cK9OO9Dkqb0MU0LJDX35Vgzs/7iCoDVu3cjYnJEHAVcAfxiD/GbSbPQ7TMRMTPS5D9mZqVxBcCqZAzwFqR5FyQ9kFsFVkrqnDXuauCTudXg1znuZTnOE5KuLpzvq5IelfS8pC92vZiksZIezuda1RlH0kuS9pf0zbxvhaTVkhbl/dMlLc5puyPPEWFm1q88EJDVNUk7SCOrNQBjgVMiYnnndLQR0SFpf2AJaUjZg0kjEh6Rj58B/AiYFhHvSGqNiDclPQQsj4hLJM0ELo6IaV2ufQnQEBE/kzQkX29znvtgSkS8keMNAx4kzeO+mDRG/oyIeFvSZaSR8X6yL78nM6ueoXuOYjaovRsRkwEkHQ/Mk3QEaejVn0s6kTQV8Ti6nzZ1GnBzRLwDEBHF+d07J21aDhzSzbGPATflB/xdEbGihzReSxrn/p48Q+Ik4JE0xDvDSZUCM7N+5QqAVUZELM6/9g8gjRl/AHBsRGzLv8obennK9/LnDrr5txQRD+cKxhnAnyVdExHzinEknUtqdbioMwhYGBFn9TItZma94j4AVhmSDgeGkKaU3Q9Ylx/+J5MewgCbgdGFwxYC50lqzOdo7cX1DgZej4g/ATeQpvwt7j8W+D5wdkS8n4OXACdIOizHGSXpU73LqZnZnrkFwOrdSEmdTe8C5kTEDkm3APdIWkmale9ZgIjYIOkRSauA+yLiUkmTgWWStgILgCv38tonAZdK2gZsIU3ZWnQR0Aosys39yyLi/NwqcKukETneD0kz3JmZ9Rt3AjQzM6sgvwIwMzOrIFcAzMzMKsgVADMzswpyBcDMzKyCXAEwMzOrIFcAzMzMKsgVADMzswr6P/XQgGP1fIkeAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plot_absolute_time(results, [2, 10], max_batch_size=max_batch_size)" + ] + }, + { + "cell_type": "markdown", + "id": "6e920708", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "#### $N_\\text{ref} \\gg N_\\text{test}$\n", + "\n", + "Now we check whether the speed improvements still hold when $N_\\text{ref} \\gg N_\\text{test}$ ($N_\\text{ref} / N_\\text{test} = 10$) and a large part of the kernel can already be computed at initialisation time of the PyTorch (but not the KeOps) detector." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "a75794e8", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "experiments = {\n", + " 'keops': {\n", + " 0: {'n_ref': 2000, 'n_test': 200, 'n_runs': 10, 'n_features': 2},\n", + " 1: {'n_ref': 5000, 'n_test': 500, 'n_runs': 10, 'n_features': 2},\n", + " 2: {'n_ref': 10000, 'n_test': 1000, 'n_runs': 10, 'n_features': 2},\n", + " 3: {'n_ref': 20000, 'n_test': 2000, 'n_runs': 10, 'n_features': 2},\n", + " 4: {'n_ref': 50000, 'n_test': 5000, 'n_runs': 10, 'n_features': 2},\n", + " 5: {'n_ref': 100000, 'n_test': 10000, 'n_runs': 10, 'n_features': 2}\n", + " },\n", + " 'pytorch': {\n", + " 0: {'n_ref': 2000, 'n_test': 200, 'n_runs': 10, 'n_features': 2},\n", + " 1: {'n_ref': 5000, 'n_test': 500, 'n_runs': 10, 'n_features': 2},\n", + " 2: {'n_ref': 10000, 'n_test': 1000, 'n_runs': 10, 'n_features': 2}\n", + " }\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "fcdd840a", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [], + "source": [ + "results = {backend: {} for backend in backends}\n", + "\n", + "for backend in backends:\n", + " exps = experiments[backend]\n", + " for i, exp in exps.items():\n", + " results[backend][i] = experiment(\n", + " backend, exp['n_runs'], exp['n_ref'], exp['n_test'], exp['n_features']\n", + " )" + ] + }, + { + "cell_type": "markdown", + "id": "27307020", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "The below plots illustrate that KeOps indeed still provides large speed ups over PyTorch. The x-axis shows the reference batch size $N_\\text{ref}$. Note that $N_\\text{ref} / N_\\text{test} = 10$." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "0a3c0d27", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAEWCAYAAAAZ9I+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd5wU9f3H8deHDtKLCAKCAQREUThRrNgIikKIAbEgtmA3sUbNL7aoMbG3WFA0qNgwKCqKGrCCCigIIigiCEc7QJAOd/f5/fGd0+W8zu7Nlffz8djH7s7Mznxmd+/ms99q7o6IiIhUblXiDkBERETip4RARERElBCIiIiIEgIRERFBCYGIiIighEBERERQQlDhmJmbWfsC1j9iZn9LeH6Bma0wsw1m1qQEx2sbHbNaSWNOlugc9iyF47SJjlW1FI7V3Mw+MLP1ZnZXqo8nRWdm15nZ43HHIZIsSghKyMwWmtk2M2uaa/kX0QWybfT8qej5gFzb3RMtPzN6fqaZZUUXmg1m9r2ZPWlmHZMZt7uf7+5/j45ZHbgb6OPudd19dWEJxc6IzvGjJO3rPTM7N3FZdA4LkrH/XMdaaGbHJBznh+hYWck+Vh6GA6uA+u5+xc7uzMxamNk4M1ua+D1NWF/TzEaa2U9mttzMLs+1/mgzm2tmm8xskpntsbMxxcHMbjSzZ4qxfW8zW5K4zN1vc/dz83uNSHmjhGDnfA+ckvPEzPYB6uSx3TfAGQnbVQMGA9/l2m6Ku9cFGgDHAJuB6WbWNRnB5vGLtjlQC/gqGfuXlNgDmOMlGEEsn1KbbOAt4KR8XnYj0CE67pHA1WbWN9pfU+C/wN+AxsA04IXixlVcZaH0SaRScHfdSnADFgL/B0xNWHYn8FfAgbbRsqei5SuARtGyE4A3gY+AM6NlZwIf5XGc14ExBcRxFbAMWAqcHR27fcKxHwbGAxsJScZTwC1Ax2iZAxuAicAH0fON0bKT8zhe1eh8VgELgIui11SL1jcAnohiSo+OVRXoDGwBsqJ9r422rxnt74foPXoEqJ1wvAHADOAnQgLVF7g12s+WaF8PRtsmnnsDYBSQASyKPqsqie91dNwfCYndcfm8v08TLqKbo2NdDbTNdc7vRec5OdrmNaAJ8GwU99Sc70O0fSfgHWANMA8YnM+xnwK2A9ui/R4TvV/3Rp/30uhxzWj73sAS4C/AcuDpAr431Uj4niYsX0ooMcp5/nfg+ejxcGBywrpdovelUwF/I9cCc6L3+UmgVsL6E6LPdm303u2b67V/Ab4EtkbxLiR8378kfEefICS1bwLrgXf55W+sN7Akj3iOIXyHtkXv7QZgZrT+LODraF8LgPNynWd2tP0GoCUheXomYf/9Ccn12ug70TnXsa+MYl9HSKRqReuaEv7O10bfiQ+Jvqu66Vaat9gDKK+3hH8u8wgXu6rRP+M9+HVCcAvwGHBBtOxFQslCURKCs4EV+cTQl3AR7Rr90xrNrxOCdcAhhNKgWjnxROvbknBhi5b9/Pp8jnk+MBdoTfiVOIkdL45jgUejeHYFPkv4x/qrcwTuAcZF+6pHuJj+I1rXM4r/2Cj+3YkuPtE/3HNz7Svx3EcBr0b7bEsopTknIY7twB+jz+0CwoXQCvqsE57v8L5FscwHfkNIROZExzuGcCEbBTwZbbsLsJhw8akG7E9Irrrkc+yfP6/o+c3AJ9F724xwIf17tK43kAn8k5A41M5rn9G2v0oIgEbRsuYJy/4AzIoe3wc8nGs/s4GTCnjfZid8Vz7ml+/e/sBK4MDoMxgWbV8z4bUzotfWTlj2CSEJ2D16/efRvmoRktobEt6LPBOC6PGNJFzMo2X9os/QgCOATUD3Avb38z74JcE+FqhOSBznAzUSjv0ZIZFoTEg8zo/W/YOQCFePboeRz3dRN91SeVOVwc57mlAdcCzhjzw9n+1GAWeYWUPCP5tXirj/pYR/IHkZTLjQzHb3jYR/ULm96u4fu3u2u28p4jELMhi4190Xu/sawj8zIDSAA44H/uzuG919JeGCPySvHZmZEX51Xubua9x9PXBbwvbnACPd/Z0o/nR3n1tYgFHVyBDgWndf7+4LgbuAoQmbLXL3ER7aAfwHaEG40JTUk+7+nbuvI/xi/c7d33X3TOAlwkULwq/ihe7+pLtnuvsXwMvAoCIe5zTgZndf6e4ZwE25ziubcFHc6u6bi3kOdaP7dQnL1hGSqpz169hR4vq8PJjwXbmVX6rYhgOPuvun7p7l7v8hlAQclPDa+6PXJp7HA+6+wt3TCb+kP3X3L6Lv9lh+eZ+Lzd3fiD5Dd/f3gbcJF+eiOBl4I/qubieUPtUGDs51Pkuj9+I1YL9o+XbC928Pd9/u7h+6uyaZkVKnurmd9zShqL0d4aKfJ3f/yMyaEaoUXnf3zeF6WKjdCcWIeWkJTE94viiPbRYX5SDF0DLXPhOPuQfhF86yhHOrUkAMzQhtLqYnbG+EX4wQfh2OL0GMTaM4EmNbRHgvcyzPeeDum6Lj16XkViQ83pzH85x97wEcaGZrE9ZXI3yPiqIlvz6vlgnPM3Yi8dsQ3dcnVMfkPF6fsL5+rtckrs9L7u9KTqx7AMPM7JKE9TXY8Vzy+t4U9X0uNjM7DriB8Gu/CuG7OauIL9/hc3H3bDNbTD7fOULpQ8653kFI5t+OvoePufvtJTgFkZ2iEoKd5O6LCHXQxxMaXBXkGeAKCkgc8jCQ8EsoL8sIF80cbfIKsRjHKoqCjrmY8Cuvqbs3jG713X3vfGJZRfgnvnfC9g08NKzM2d9v8omjoPNaRfjVldgCvg35l94UJpnv4WLg/YTzbeihx8IFRXz9Un59XksTnpc4Vnf/kfD5dktY3I1fGp1+lbjOzHYhfD4FNUrN/V3JiXUxcGuu96GOuz+XGFLJzgQIxfc/N/CNSo2a5bdvM6tJKKm5k1Bl0pCQjFpe2+dhh88lKv1qTRG+c1Ep1hXuviehHcLlZnZ0Ya8TSTYlBMlxDnBUVGxfkPsJVQsfFLSRmVU1s3Zm9gCh7vKmfDZ9ETjTzLqYWR3Cr5udtQIoqC//i8ClZtbKzBoB1+SscPdlhGLWu8ysvplVMbPfmNkRCftuZWY1ou2zgRHAPWa2K4CZ7W5mv422fwI4K+rqViVa16mwOKNqgBeBW82sXtQ17nJCQlYShb0nxfE60NHMhppZ9eh2gJl1LuLrnwP+z8yaRa3+r6eY52VmtQhtDABqRs9zjIr23yh6r/9IaMcAoUi+q5mdFL3meuDLQqpxLoq+K40JpWM5vRJGAOeb2YEW7GJm/cysoOqH4vgGqBXtszqhUWnNhPUrgLZmlvM/sEa0PgPIjEoL+uTavomZNcjneC8C/aLvanVC4r+V0MajQGZ2gpm1j5KIdYQGs9lFPVGRZFFCkARRveO0Imy3xt3/V0D9YC8z20Bomf4eoTj2AHfPs9jS3d8ktDKfSGjANLEk8edyI/AfM1trZoPzWD8CmADMJDToyl0qcgbhn2tOy/IxhPpRovi+Apab2apo2V+i2D8xs58ILcX3is7vM0Lju3sI/yjf55dfYfcBfzCzH83s/jzivITwK3EBofHmaGBk0d6CX/kH4SK51syuLOE+gPBrkHChGUL4VbmcXxoBFsUthO5+XxKKsz+PlhVHTo8JCA1EE+vobyD05lhEeL/vcPe3otgzCN0VbyV8tgeST/uQBKMJSeKCaL+3RPuaRkg2Hoz2NZ/Q2DMporYcFwKPE36lbyQ0+s3xUnS/2sw+jz6XSwkX9h+BUwmNXXP2N5eQjC2IvgeJVRu4+zzgdOABQgnVicCJ7r6tCOF2IHzvNwBTgH+7+6TinbHIzjO1XRGRVDCzhYSeIO/GHYuIFE4lBCIiIqKEQERERFRlICIiIqiEQERERKhgAxM1bdrU27ZtG3cYIiLlxvTp01e5e7PCt5SKrkIlBG3btmXatEJ7/4mISMTM8hrhVCohVRmIiIiIEgIRERFRQiAiIiIoIRARERGUEIiIiAhKCERERAQlBCIiIoISAhERESGFCYGZjTSzlWY2O5/1V5nZjOg228yyzKxxtG6hmc2K1pX7kYa+/x569YL33487EhERkbylsoTgKaBvfivd/Q5338/d9wOuBd539zUJmxwZrU9LYYylompV6N4dGjaMOxIREZG8pWzoYnf/wMzaFnHzU4DnUhVL3Nq0gYceijsKERGR/MXehsDM6hBKEl5OWOzA22Y23cyGF/L64WY2zcymZWRkpDLUEtu4EbZvjzsKERGR/MWeEAAnAh/nqi441N27A8cBF5nZ4fm92N0fc/c0d09r1qxsTtj1179CGQ1NREQEKBsJwRByVRe4e3p0vxIYC/SMIa6kSU+H3XaLOwoREZH8xZoQmFkD4Ajg1YRlu5hZvZzHQB8gz54K5cXSpbD77nFHISIikr+UNSo0s+eA3kBTM1sC3ABUB3D3R6LNBgJvu/vGhJc2B8aaWU58o939rVTFWRrS0+Gww+KOQkREJH+p7GVwShG2eYrQPTFx2QKgW2qiKn3Z2SohEBGRsi9lCYEEmZlw881hYCIREZGySglBitWoAddcE3cUIiIiBSsLvQwqtLVrYdEiyMqKOxIREZH8KSFIsRdfhLZtQzsCERGRskoJQYotXQpmGodARETKNiUEKZaeDrvuCtWrxx2JiIhI/pQQpJi6HIqISHmghCDF0tOhZcu4oxARESmYuh2m2PXXwy67xB2FiIhIwZQQpNjvfx93BCIiIoVTlUEKbdgAH30EP/0UdyQiUlG98QZcdRW4xx2JlHdKCFJo5swwqdHkyXFHIiIVjTvceSeceCL873+wcWPhrxEpiBKCFEpPD/fqZSAiybR1K5x9digZOOkk+PBDqFs37qikvFNCkEJKCEQk2VauhKOOgqeeghtugBdeUMNlSQ41Kkyh9HSoVQsaNYo7EhGpCL78MlQRZGSERGDw4LgjkopEJQQplJ4eSgfM4o5ERMq7V16Bgw8OU6p/+KGSAUk+lRCk0F/+EjJ5EZGScofbb4frroMDDgiJgQY7k1RQQpBC++0XdwQiUp5t2QLnngvPPgunnAJPPAG1a8cdlVRUqjJIEfdQx/fdd3FHIiLl0bJlcMQRIRm49dZwr2RAUillCYGZjTSzlWY2O5/1vc1snZnNiG7XJ6zra2bzzGy+mV2TqhhTae1aGDIEXn017khEpLz5/PNQPfDVV/Df/4bqArVFklRLZQnBU0DfQrb50N33i243A5hZVeAh4DigC3CKmXVJYZwpoS6HIlISL70Ehx4KVarAxx/DwIFxRySVRcoSAnf/AFhTgpf2BOa7+wJ33wY8DwxIanClYOnScK/GPyJSFO5w002h98D++8PUqdCtW9xRSWUSdxuCXmY208zeNLO9o2W7A4sTtlkSLcuTmQ03s2lmNi2jDDXpVwmBiBTVpk2hivHGG+GMM2DiRGjePO6opLKJMyH4HNjD3bsBDwCvlGQn7v6Yu6e5e1qzZs2SGuDOyEkIVEIgIgVZsiTMefLSS3DHHWEEwpo1445KKqPYEgJ3/8ndN0SPxwPVzawpkA60Tti0VbSsXPnjH8NMh7VqxR2JiJRVn30GPXvCN9/AuHFw5ZVqPCjxiS0hMLPdzMJX38x6RrGsBqYCHcysnZnVAIYA4+KKs6SaN4dDDok7ChEpq0aPhsMPDz8apkyBE06IOyKp7FI2MJGZPQf0Bpqa2RLgBqA6gLs/AvwBuMDMMoHNwBB3dyDTzC4GJgBVgZHu/lWq4kyVUaOgTRvo3TvuSESkLMnOhr/9DW67LSQEL78MTZvGHZUIWLgGVwxpaWk+bdq0uMMAoEUL6NcPHn887khEpKzYsAGGDg3DD597Ljz0ENSoEW9MZjbd3dPijULKAg1dnALbt8OKFephICK/WLQI+veH2bPh3nvh0kvVXkDKFiUEKbBiRehTrB4GIgIweXIYYGjrVhg/Hn7727gjEvm1uMchqJA0BoGI5PjPf+DII6F+ffjkEyUDUnYpIUiBnFEKlRCIVF5ZWXD11XDmmWGcgU8/hU6d4o5KJH+qMkiBfv1gwQJVGYhUVj/9BKeeCm+8ARdeGNoMVK8ed1QiBVNCkAI1akC7dnFHISJxWLAgNB6cOzf0IrjwwrgjEikaJQQp8OyzsHEjDB8edyQiUprefx9OOimMNfD223DUUXFHJFJ0akOQAk8+GcYjF5HKY8QIOOaYMMjQp58qGZDyRwlBCqSnq/2ASGWRmQl//nMoETz66NCToEOHuKMSKT4lBCmQnq4eBiKVwdq1oRHxffeFpOD116Fhw7ijEikZtSFIsvXrw00JgUjF9u23cOKJoRHhiBFhKGKR8kwJQZKtWBHuVWUgUnG9+y4MHgxVqoTHhx8ed0QiO08JQZK1bw9btsQdhYikyr//HeYh6NQJXntNXYyl4lAbghSoWTPcRKTi2L49jClw0UVw3HFhfgIlA1KRKCFIsldegUsuCcOWikjFsGYN9O0LDz8chiN+5ZUwN4FIRaKEIMkmToRRo6Bq1bgjEZFk+PprOPBA+OijML7IP/+pv2+pmNSGIMnU5VCk4njrLTj5ZKhVCyZNgoMPjjsikdRRCUGSLV2qHgYi5Z17mJCoX7/QTmDqVCUDUvEpIUgylRCIlG/btoVRBy+7DAYMCFUFbdrEHZVI6qUsITCzkWa20sxm57P+NDP70sxmmdlkM+uWsG5htHyGmU1LVYzJ5h7uW7eONw4RKZmMjDAfweOPw//9H4wZA3Xrxh2VSOlIZRuCp4AHgVH5rP8eOMLdfzSz44DHgAMT1h/p7qtSGF/SmcEPP/ySGIhI+TF7dhh5cNkyGD0aTjkl7ohESlfKSgjc/QNgTQHrJ7v7j9HTT4BWqYqltJnFHYGIFMfrr0OvXrB1K3zwgZIBqZzKShuCc4A3E5478LaZTTez4QW90MyGm9k0M5uWkZGR0iAL89FHMHAgLFoUaxgiUkTucMcd0L8/7LVXaDzYs2fcUYnEI/aEwMyOJCQEf0lYfKi7dweOAy4ys3xHCnf3x9w9zd3TmjVrluJoCzZ7dhiwpJo6c4qUeVu2wJlnhoGGBg0KJQNqECyVWawJgZntCzwODHD31TnL3T09ul8JjAXKRc6enh4mO2nePO5IRKQgK1bAUUeFQcRuvhmefx7q1Ik7KpF4xfZb1szaAP8Fhrr7NwnLdwGquPv66HEf4OaYwiyWpUtDMqASApGya8aMUEWwahW89BL84Q9xRyRSNqTs0mVmzwG9gaZmtgS4AagO4O6PANcDTYB/W2iFl+nuaUBzYGy0rBow2t3fSlWcyaQxCETKtv/+F4YOhcaNQ5uf7t3jjkik7EhZQuDuBbbTdfdzgXPzWL4A6PbrV5R9jRpBqwrTV0Kk4nCH224LYwsceCCMHQstWsQdlUjZosLtJHruubgjEJHcNm+Gc84Jf5+nnRYGHapVK+6oRMqe2HsZiIikytKlcMQRodHgP/4BTz+tZEAkP0oIkmTRIkhLg//9L+5IRARg2jQ44ACYMydUEVxzjQYNEymIEoIk+eEHmD4dsrLijkREXngBDjsMqleHyZPDJEUiUjAlBEmSnh7u1ctAJD7Z2XDDDTBkCPToAZ99BvvuG3dUIuWDGhUmiRICkXht3AjDhsHLL8NZZ8HDD0PNmnFHJVJ+KCFIkvT0MNJZgwZxRyJS+SxeHKoFZsyAO++Eyy9XewGR4lJCkCS77w7HHad/QiKl7ZNP4He/g02bwqyFxx8fd0Qi5ZMSgiS54oq4IxCpfJ55Bs49NyTkEydCly5xRyRSfqlRoYiUO9nZcO21YRjiXr1C40ElAyI7RwlBErhDu3Zw771xRyJS8a1fDwMHwu23w/DhMGECNGkSd1Qi5Z+qDJJgzRpYuFDtB0RSbeHCMFPhV1/B/ffDxRfr704kWZQQJMHSpeG+Zct44xCpyD76CH7/e9i2Dd58E/r0iTsikYpFVQZJoDEIRFLrySfhqKPCjKKffqpkQCQVlBAkQU5CoBICkeTKygo9eM4+O0xS9MknsNdecUclUjGpyiAJdt89NHLS/OoiybNuHZxySqgeuOQSuPtuqKb/WCIpU+ifl5n1Ak4HDgNaAJuB2cAbwDPuvi6lEZYDffuGm4gkx3ffwYknwrffwiOPwHnnxR2RSMVXYEJgZm8CS4FXgVuBlUAtoCNwJPCqmd3t7uNSHWhZlp0NVVT5IpIU770HJ50UHr/9Nhx5ZKzhiFQahZUQDHX3VbmWbQA+j253mVnTlERWjhxwQKjXHD067khEyrfHHoOLLoIOHWDcOGjfPu6IRCqPAn/X5iQDZraLmVWJHnc0s/5mVj1xm7yY2UgzW2lms/NZb2Z2v5nNN7Mvzax7wrphZvZtdBtWkpMrLUuWQN26cUchUn5lZsKll4aqgWOPhSlTlAyIlLaiFnR/ANQys92Bt4GhwFNFeN1TQEG168cBHaLbcOBhADNrDNwAHAj0BG4ws0ZFjLVUbd8OK1eqy6FISf34Y5iQ6IEHQo+C117TrKEicShqQmDuvgn4PfBvdx8E7F3Yi9z9A2BNAZsMAEZ58AnQ0MxaAL8F3nH3Ne7+I/AOBScWsVm2LNyry6FI8X3zDRx0UGg38MQTYeriqlXjjkqkcipyQhD1NjiN0LsAIBl/trsDixOeL4mW5bc8r8CGm9k0M5uWkZGRhJCKR4MSiZTMO+/AgQeGob8nTgxjDYhIfIqaEPwJuBYY6+5fmdmewKTUhVV07v6Yu6e5e1qzZs1K/fhNmoS6z86dS/3QIuWSOzz4IBx3HLRqBVOnwqGHxh2ViBRpmI+o6P+DhOcLgEuTcPx0oHXC81bRsnSgd67l7yXheEnXsSPcd1/cUYiUD9u3h0GGHn00TFL0zDNQr17cUYkIFFJCYGYjzGyffNbtYmZnm9lpO3H8ccAZUW+Dg4B17r4MmAD0MbNGUWPCPtGyMmftWti6Ne4oRMq+1avDHASPPgrXXANjxyoZEClLCisheAj4W5QUzAYyCAMTdQDqAyOBZ/N7sZk9R/il39TMlhB6DuR0V3wEGA8cD8wHNgFnRevWmNnfganRrm5294IaJ8bm4ovh44/h++/jjkSk7JozJ4w8mJ4OTz8Np58ed0QikluBCYG7zwAGm1ldII1fhi7+2t3nFbZzdz+lkPUOXJTPupGEhKNMW7pUDQpFCjJ+PAwZAnXqhN4EBx0Ud0QikpeitiHYQBmtw49bejp06xZ3FCJljzvccw9cdRXsu28YebB168JfJyLx0Aj8O8E9JAQqIRDZ0datcM45YaChgQPho4+UDIiUdUoIdsL69bBxowYlEkm0ciUccww8+ST87W/w4ouwyy5xRyUihSnW7OJmVicasVAAM7jjDs3GJpJj1qzQeHDFCnj+eTj55LgjEpGiKlIJgZkdbGZzgLnR825m9u+URlYO1KsHV14JPXrEHYlI/MaNg4MPDmMNfPihkgGR8qaoVQb3EOYXWA3g7jOBw1MVVHmxciV89x1kZ8cdiUh83OH22+F3vwsjdk6dCmlpcUclIsVV5DYE7r4416KsJMdS7jzxRJiidcuWuCMRiceWLXDGGXDttaFE4P331aZGpLwqakKw2MwOBtzMqpvZlcDXKYyrXEhPh4YNQ/9qkcpm+XLo3TsMP/z3v8Po0VC7dtxRiUhJFbVR4fnAfYQZB9OBt8lnQKHKRIMSSWX1xRdhLoI1a+Dll+H3v487IhHZWUUdmGgVYepjSaAxCKQyevnlUE3QpEkYtnu//eKOSESSoUgJgZm1Ay4B2ia+xt37pyas8iE9HfbeO+4oREqHe6gauOGGMPzw2LGw225xRyUiyVLUKoNXgCeA1wC1qY/cc48aUEnlsGkTnH02vPBCKB149FGoVSvuqEQkmYqaEGxx9/tTGkk5NGhQ3BGIpF56OgwYAJ9/Dv/8Z5ibwCzuqEQk2YqaENxnZjcQGhNuzVno7p+nJKpyYPVqmD0bunfXnO5ScU2dGpKB9evh1VfDKIQiUjEVNSHYBxgKHMUvVQYePa+UPvooDMSiQVikonruuVBNsNtuMGEC7LNP3BGJSCoVNSEYBOzp7ttSGUx5kp4e7tXLQCqa7OzQcPCWW+Cww0KvgmbN4o5KRFKtqAMTzQYapjKQ8mbpUqhaFXbdNe5IRJJn48bQNuaWW8L0xe++q2RApLIoaglBQ2CumU1lxzYElbbbYXp6KEqtWjXuSESS44cfwmBDs2aFHjR/+pMaD4pUJkVNCG5IaRTlkAYlkopkyhQYOBA2b4Y33oC+feOOSERKW1FHKny/JDs3s76EIY+rAo+7++251t8DHBk9rQPs6u4No3VZwKxo3Q9lrTTi9tvDP0+R8m7UKPjjH6F1a5g0KcxYKCKVT4EJgZl95O6Hmtl6Qq+Cn1cB7u71C3htVeAh4FhgCTDVzMa5+5ycbdz9soTtLwH2T9jFZncvs4Oidu8edwQiOycrC667Dv71LzjqKHjxxTAcsYhUToU1KtwFwN3ruXv9hFu9gpKBSE9gvrsviHonPA8MKGD7U4Dnihx5jLZuhWefhYUL445EpGTWrw9VBP/6F5x/Prz1lpIBkcqusITAC1lfkN2BxQnPl0TLfsXM9gDaARMTFtcys2lm9omZ/S6/g5jZ8Gi7aRkZGTsRbtEtXgynnx7mfhcpb77/Hg4+GMaPh4cegocfhurV445KROJWWBuCXc3s8vxWuvvdSYpjCDDG3bMSlu3h7ulmticw0cxmuft3ecTwGPAYQFpa2s4kMEWmMQikvPrgAzjpJMjMDKUCxxwTd0QiUlYUVkJQFagL1MvnVpB0oHXC81bRsrwMIVd1gbunR/cLgPfYsX1BrJQQSHn0xBMhAWjSBD77TMmAiOyosBKCZe5+cwn3PRXoEE2dnE646J+aeyMz6wQ0AqYkLGsEbHL3rWbWFDgE+FcJ40g6JQRSnmRmhgmJ7r0X+vQJMxY21DBjIpJLYQlBiYclcfdMM7sYmEAoaRjp7l+Z2c3ANHcfF206BHje3ROL+zsDj5pZNqEU4/bE3glxS0+HunWhfmHNKkVitm4dDBkSqgcuvRTuuguqFXX0ERGpVGzH63Culf/muWUAACAASURBVGaN3X1NKcazU9LS0nzatGkpP86KFSEpUNdDKcvmzw+zE86fHxoPDh8ed0RSFpnZdHfXFG1ScAlBeUoGSlPz5uEmUlZNnAh/+ANUqRLmIzjiiLgjEpGyrqiTG0mCf/8bPv447ihE8vbww6GtQIsWofGgkgERKQolBMWUnQ1//jO89lrckYjsaPt2uPhiuPDCMBfBlCmw555xRyUi5YUSgmJavTr8423ZMu5IRH6xZg0cd1xoK3DllfDqq2r0KiLFo/bGxaQuh1LWzJ0bGg/+8AM8+SSceWbcEYlIeaSEoJiWLg33SgikLJgwAU4+GWrUCA0JDzkk7ohEpLxSlUEx5ZQQqMpA4uQO990Hxx8Pe+wBU6cqGRCRnaOEoJjOPDMUzaqEQOKybRucd15o3Nq/f+jxsscecUclIuWdEoJiql4dWreGqlXjjkQqo1WrQpfCESPguuvg5ZfDqJkiIjtLbQiK6eGHoVYtOOusuCORyuarr0LjwaVL4dln4dRfzQwiIlJyKiEopkcfhf/+N+4opLJ5/XXo1Qs2b4b331cyICLJp4SgmNLT1X5ASo873HlnaCvQoUNoPHjggXFHJSIVkRKCYti6NdThKiGQ0rB1a6iauuqqMC/Bhx9Cq1ZxRyUiFZUSgmJYtizcq8uhpNrKlXDUUfCf/8CNN8Lzz0OdOnFHJSIVmRoVFsPKlWH2OJUQSCrNnBmqCDIy4MUXYdCguCMSkcpACUEx9OwZinFFUuWVV+D006Fhw1BF0KNH3BGJSGWhKoNiqlYt3ESSyR1uuw0GDoS99w6NB5UMiEhpUkJQDE89BZddFncUUtFs3hxKBf7619Cd8L33oEWLuKMSkcpGCUExvP02jBsXdxRSkSxbBr17w+jRoYTgmWegdu24oxKRyiilCYGZ9TWzeWY238yuyWP9mWaWYWYzotu5CeuGmdm30W1YKuMsqqVL1cNAkmf6dDjggDAC4dixcO21YBZ3VCJSWaWsNtzMqgIPAccCS4CpZjbO3efk2vQFd78412sbAzcAaYAD06PX/piqeIsiPV31upIcL70Ew4ZBs2ZhcqJu3eKOSEQqu1SWEPQE5rv7AnffBjwPDCjia38LvOPua6Ik4B2gb4riLBL3UEKgLoeyM7Kz4aabYPBg6N49NB5UMiAiZUEqE4LdgcUJz5dEy3I7ycy+NLMxZta6mK/FzIab2TQzm5aRkZGMuPO0aRM0ahRmOhQpiU2bYMiQMNDQsGHwv//BrrvGHZWISBB3B7rXgOfcfauZnQf8BziqODtw98eAxwDS0tI8+SEGu+wCS5akau9S0S1ZAgMGwBdfwB13wBVXqL2AiJQtqSwhSAcSf0+3ipb9zN1Xu3vOUD+PAz2K+lqR8uLTT0PjwW+/hddegyuvVDIgImVPKhOCqUAHM2tnZjWAIcAOnfbMLLG3dX/g6+jxBKCPmTUys0ZAn2hZbF5/PcxFv3p1nFFIeTN6NBxxROhKOGUK9OsXd0QiInlLWZWBu2ea2cWEC3lVYKS7f2VmNwPT3H0ccKmZ9QcygTXAmdFr15jZ3wlJBcDN7r4mVbEWxcyZISnQBDNSFNnZ8Le/hbEFjjgCxoyBpk3jjkpEJH8pbUPg7uOB8bmWXZ/w+Frg2nxeOxIYmcr4iiM9HRo31qAxUjD3MNLgP/4B77wDf/wjPPgg1KgRd2QiIgWLu1FhuTF7NrRvH3cUUlZt3AjPPgsPPBC+K02ahETgwgvVXkBEygcNXVwE27fDtGnQq1fckUhZ8/33oZFgq1Zw3nlh4qsnnoDFi+Gii5QMiEj5oRKCIsjIgLS0UBcs4g4TJ8L994deA1WqwEknwSWXwCGHKAkQkfJJCUERtGwJH3wQdxQStw0b4OmnQ1XAnDlh2OHrroPzzw8lBCIi5ZkSgiLIyoKqVeOOQuLy3Xfw0EMwciSsWxfms3jqKTj5ZKhVK+7oRCqG6dOn71qtWrXHga6oOjsVsoHZmZmZ5/bo0WNlXhsoISiCzp1h0CC49da4I5HSkp0N774bqgXGjw8J4aBBoVrgoINULSCSbNWqVXt8t91269ysWbMfq1SpkrJRZyur7Oxsy8jI6LJ8+fLHCeP+/IoSgkKkp4cR5po1izsSKQ3r18N//hOqBebNg+bNw3gC552nqa9FUqyrkoHUqVKlijdr1mzd8uXLu+a3jRKCQkyZEu4PPjjeOCS1vv02JAFPPhmSgp49Q3uBQYOgZs24oxOpFKooGUit6P3NtzpGCUEhpkwJ9cT77Rd3JJJs2dkwYUKoFnjrLahePUxLfMklcOCBcUcnIlK61HCjEJMnhy6HGmmu4li3Du67Dzp1guOPhxkz4Kab4Icf4JlnlAyIVEbz5s2r0aFDh73jjiM//fv3b9e2bduuHTp02HvQoEFtt27dmvSWTEoICjFkCAwfHncUkgxz58LFF4cugn/+c5hbYPRoWLQIrr8edtst7ghFRPJ22mmnrVmwYMHsefPmfbVlyxa79957kz47ihKCQvzpTzB0aNxRSEllZ4dJqX7729BbZMQI+P3vYerUUPpzyikq/RGRHc2ZM6dG586du7z//vt1MjMzOe+881p17dq1c8eOHbvccccdTQGys7M577zzWnXo0GHvjh07dhkxYkQjgNdff71eWlraXr17927ftm3brqeeemqbrKwsMjMzOemkk9rmbH/TTTftWpyYTj755HVVqlShSpUqpKWlbVyyZEnS/3OpDUEBvvsO6tWDXYv1sUlZsHZtGDfgoYdgwYLQQ+CWW8JkQ/o8Rcq2s8+m9ezZJHVu2a5d2TRyJIsL227mzJk1hwwZ8puRI0d+36tXr8133nln0wYNGmTNnj37682bN9sBBxzQ6cQTT/zpk08+qTNr1qzaX3/99VfLli2r1rNnz859+vTZADBr1qxdvvjii9kdO3bcdvjhh3cYNWpUo/bt229dtmxZ9W+//fYrgFWrVpVodJutW7faCy+80OTuu+8u9FyKSwlBAa6+OtQvf/dd3JFIUc2ZEyYYGjUKNm2CQw8NMw8OHBgaDYqI5GfNmjXVfve737UfM2bMdz169NgC8O6779afO3dunXHjxjUCWL9+fdU5c+bU+vDDD+sNHjx4TbVq1WjdunXmgQceuOGjjz6q06BBg+x99tlnY5cuXbYBDB48eM2HH35Y94QTTvhp8eLFNYcNG9b6xBNPXDdw4MCfShLjsGHD2hx00EEb+vbtuyF5Zx4oIciHeyhSPvrouCORwmRlhWqBBx6A//0vdBM89dTQW2D//eOOTkSKqyi/5FOhXr16WS1bttw2adKkujkJgbvbXXfd9cNJJ520wwX8jTfeaJDffizXyGVmRrNmzbJmz549Z+zYsfUfeeSRZi+88ELjl156aWHONpmZmXTt2rULQN++fdfee++9S3Pv94orrmixatWqahMmTEjJz1S1IcjHokWwfLnGHyjL1qyBO+4I01L/7ndhIKHbboMlS0J1gZIBESmO6tWr+5tvvvndc8891+SRRx5pDHDssceue/jhh5vltOr/8ssva/70009VDj/88PVjxoxpnJmZydKlS6t99tlndQ877LCNEKoM5s6dWyMrK4sxY8Y0Puyww9YvW7asWlZWFmeeeebaf/zjH+mzZs3aoUqkWrVqzJ07d87cuXPn5JUM3H333U0nTpzY4JVXXllQNUVj6auEIB+TJ4d7TXlc9syaFUoDnnkGNm+Gww+HO++EAQPC9MMiIiVVv3797AkTJszv3bt3x3r16mVddtllqxYuXFhzn3326ezu1rhx4+3jx4//bujQoWsnT55ct3Pnznubmd90001L2rRpk/nll1/StWvXjeeff36bhQsX1jr44IN/Gjp06NrPPvus9jnnnNM2OzvbAG6++eYlxYnr6quv3qNFixZb09LSOgOccMIJP955553Lknnu+veZjylTYJddYJ994o5EADIzYdy4kAi8914YLOr000M3wm7d4o5ORMq7vfbaa1tOg7+mTZtmzZ49++ucdQ8++GA6kJ77NY8++ugS4FcX9nr16mVNmjRpfuKyXr16bZ4zZ87XubctqszMzOklfW1RKSHIx5//DH376hdn3Favhscfh3//OwwctMce8M9/wjnnQJMmcUcnIlJxpPRyZ2Z9gfuAqsDj7n57rvWXA+cCmUAGcLa7L4rWZQGzok1/cPc8Z2dKld/8JtwkHjNmhNKA0aNhyxY48sgwuuCJJ2oqahEpu0444YT1J5xwwvq44yiJlCUEZlYVeAg4llCkMtXMxrn7nITNvgDS3H2TmV0A/As4OVq32d1jmUHg669DG4LBg8M4BFI6tm+HV14JicCHH0KdOjBsWKgW6Jrv/FwiIpIMqexl0BOY7+4L3H0b8DwwIHEDd5/k7puip58ArVIYT5G98gqce264QEnqZWSE3gHt2oUkbMmS0EhwyRJ45BElAyIipSGVVQa7ww59SZcABU0bcw7wZsLzWmY2jVCdcLu7v5LXi8xsODAcoE2bNjsVcI7Jk8PEN40bJ2V3ko/p00NpwPPPw9atcMwxoa1Av36qFhARKW1losmcmZ0OpAFHJCzew93TzWxPYKKZzXL3Xw3G4O6PAY8BpKWl7fRc2u6hh0H/Um2xUHls3w4vvxwSgcmTQ0+Oc84J1QKdO8cdnYhI5ZXKKoN0oHXC81bk0W3DzI4B/gr0d/etOcvdPT26XwC8B5TKMDPffhtatmtAouRasQL+/ndo2zZMKLRiBdxzD6Snh/kGlAyISHlzzTXXJG2O1Msvv7zl9ddf37ykr588eXLt/fbbr1P79u13mGypOFKZEEwFOphZOzOrAQwBxiVuYGb7A48SkoGVCcsbmVnN6HFT4BAgsTFiysyYEe41IFFyTJ0KZ5wBbdqEKYb32ScMM/zNN6FrZ4N8B/8UESnb7r///hbF2T47O5usrKyUxFK3bt3sp59++vv58+d/9fbbb3973XXXtS7uBEopSwjcPRO4GJgAfA286O5fmdnNZpZTIH8HUBd4ycxmmFlOwtAZmGZmM4FJhDYEpZIQDB4cfr3qF2vJbdsWugsedBD07Aljx8J558HcufDWW6GNQBUNmi0iZci8efNqtGvXbu/+/fu323PPPffu27fvnuvXr68ybty4esccc8zPndDHjh1b/9hjj/3NhRdeuPvWrVurdOrUqUv//v3bAdx4443NO3TosHeHDh32vvnmm3fN2W/btm27Dhw4sG3Hjh33/u6772qMGTOmfpcuXTrvtddeXXr16tUxZ99ff/117Z49e+7VqlWrfW655ZZizcu67777bt1nn322ArRt23Z748aNM5ctW1asZgEpbUPg7uOB8bmWXZ/w+Jh8XjcZiG2MQE2PWzLLlsGjj4bb8uXQsSPcf3/oOli/ftzRiUh50rMne+Ve9vvfs+aaa8hYv54qRx9Nh9zrTz+dVZdeyuply6g2YAA7jCTz2WfMK+yYCxcurPXoo48u7NOnz8ZBgwa1veOOO5rdeOONK/70pz+1Wbp0abWWLVtmjhw5sslZZ5216tRTT1331FNP7Tp37tw5AB9++GGd0aNHN5k+ffrX7k6PHj06H3300eubNm2a9cMPP9R84oknvj/66KMXLl26tNrFF1/c9r333pvbqVOnbStWrPj5V/z8+fNrTZ48ed7atWurdu7cuetVV12VUbNmzWK3jZs0aVKd7du3W5cuXbYWvvUv9Dstwbp1YZrcnHkMpHDu8MkncNppYRTBm26C7t3hzTfDeA6XXKJkQETKh912221bnz59NgIMHTp09eTJk+tWqVKFwYMHrx4xYkTjVatWVf3888/rDho0aF3u17733nt1jz/++LX169fPbtCgQXa/fv1+nDRpUj2AFi1abDv66KM3Rtvt0rNnz/WdOnXaBtC8efOf6xD69Omztnbt2t6iRYvMxo0bb1+yZEmxf7QvWrSo+llnnbXniBEjFhZ3EqQy0cugrPj00zAGwYUXxh1J2bd1K7z4YigBmDYtXPQvvBAuugg6/CpvFxEpnoJ+0derR3ZB61u0ILMoJQK55TVtMcAFF1ywul+/fu1r1arlJ5544o/Vq1cv1n7r1KmTXZTtEksDqlatSmZm5g4BjRo1quFtt93WEuCxxx5bePjhh29KXL9mzZoqxx13XPsbbrghPScBKQ6VECSYMgXM4MCCRkuo5NLT4W9/C40EzzgDNmwIvQSWLIF771UyICLl17Jly2q8++67uwA8++yzjQ8++OANEOrkmzdvvv2uu+5qMXz48FU521erVs1zpkU+8sgjN4wfP77h+vXrq/z0009Vxo8f3+jII4/81RDGvXv33vjZZ5/Vmzt3bg2AxCqDwpxxxhlrc6ZIzp0MbNmyxfr169d+yJAhq88666wfS3L+SggSTJ4cRsVTEfeO3OHjj2HIkNBt8NZbQ9L09tswZ04oGdAQzyJS3rVt23bLAw88sOuee+6599q1a6tdeeWVGTnrhgwZsrpFixbbunfvviVn2WmnnZbRuXPnLv3792936KGHbjr11FNXd+/evXOPHj06Dx06NOOQQw7ZnPsYLVu2zLz//vsXDhw4sP1ee+3VZeDAgXsmI/aRI0c2mjp1at3Ro0c37dSpU5dOnTp1mTx5cu3i7MPcd3osnzIjLS3Np02bVqLXZmdDo0ahj/wjjyQ5sHJqy5YwiuD998MXX4QuguecE6oF9kzKV1hE4mZm0909Le44Zs6cubBbt26rCt8yNebNm1fjhBNO6JAzBXJuZ5xxRpv9999/02WXXRZbjMkwc+bMpt26dWub1zq1IYisWhW6Gh5+eNyRxG/xYnj4YRgxIrwvXbqEJOn008PIgiIilcnee+/duXbt2tmPPvro4sK3Lr+UEER23TW0lq+s3OGjj0JpwNix4Xn//qGXwJFHhrYVIiIV1V577bUtv9KBr7766uvSjicOSggi7pXzord5cxhE6IEHYObMUG1y+eWhXUDbtnFHJyKVSHZ2drZVqVKl4tRjlzHZ2dkG5NvjQY0KI/vvD9dcE3cUpWfRIvjLX6BVqzDVc3Y2PPZY6C3wr38pGRCRUjc7IyOjQXTRkiTLzs62jIyMBsDs/LZRCQFhMqOZM0Mr+orMHd5/P1QLvPpqWDZwYKgWOPzwyllCIiJlQ2Zm5rnLly9/fPny5V3Rj9VUyAZmZ2ZmnpvfBkoI+KXtQEWd0GjjRnj22VAtMHs2NGkCV18NF1wQxhMQEYlbjx49VgKaeD5GSggI4w9UrQoHHBB3JDtv+/YwNsDnn/9ymzEDNm2C/faDJ54IXStrF6t3qoiIVHRKCAgjFO6/P9SpE3ckxbN5M8yaFS76X3wR7r/8Msw2CFC3bjivc8+FQYPgkENULSAiInlTQgCceGLZ71+/fn1o55D4y3/OHMiZWrtRozCp0J/+FO67d4f27TXNsIiIFI0SAuCyy+KOYEdr1vzyiz/n/ptvQqNAgObNoUePME5AzsV/jz30619EREpOCUHMVqzY8Vf/55/DwoW/rG/TJlzwTzvtl4t/ixaxhSsiIhWUEoJS4h6GBE688H/xBSxd+ss2HTpAz55w/vnhwr///tC0aXwxi4hI5aGEIMnWrIH58+Hbb8N9zuNvvoEfowkpq1QJ8yYcffQvv/q7dQuTB4mIiMRBCUEJrF6940U/8eK/Zs0v25lB69ahcd/gwbDvvuHiv+++5a9Hg4iIVGwpTQjMrC9wH1AVeNzdb8+1viYwCugBrAZOdveF0bprgXOALOBSd5+QylgBMjMhIwOWLYPly8Mt5/GyZfDDD+Gin/NLP8QZ6vlzLvrt24ei//btwxTBtWqlOmoREZGdl7KEwMyqAg8BxwJLgKlmNs7d5yRsdg7wo7u3N7MhwD+Bk82sCzAE2BtoCbxrZh3dPSvZcbpDWhqkp4dkIDuPaR8aNoTddgvj/g8ZsuNFv107XfRFRKT8S2UJQU9gvrsvADCz54EBQGJCMAC4MXo8BnjQzCxa/ry7bwW+N7P50f6mJDtIM+jSJXTj22230II/8b55c43qJyIiFV8qE4LdgcUJz5cAB+a3jbtnmtk6oEm0/JNcr909r4OY2XBgOECbEg7M//TTJXqZiIhIhVHux7Fz98fcPc3d05o1axZ3OCIiIuVSKhOCdKB1wvNW0bI8tzGzakADQuPCorxWREREkiSVCcFUoIOZtTOzGoRGguNybTMOGBY9/gMw0d09Wj7EzGqaWTugA/BZCmMVERGp1FLWhiBqE3AxMIHQ7XCku39lZjcD09x9HPAE8HTUaHANIWkg2u5FQgPETOCiVPQwEBERkcA8Z8acCiAtLc2nTZsWdxgiIuWGmU1397S445D4lftGhSIiIrLzlBCIiIiIEgIRERGpYG0IzCwDWFTIZk2BVaUQTlmj865cdN6Vy86c9x7urkFcpGIlBEVhZtMqYwManXflovOuXCrreUtyqcpARERElBCIiIhI5UwIHos7gJjovCsXnXflUlnPW5Ko0rUhEBERkV+rjCUEIiIikosSAhEREak8CYGZ9TWzeWY238yuiTuekjCz1mY2yczmmNlXZvanaHljM3vHzL6N7htFy83M7o/O+Usz656wr2HR9t+a2bCE5T3MbFb0mvvNzEr/TPNmZlXN7Aszez163s7MPo1ifSGaVZNolswXouWfmlnbhH1cGy2fZ2a/TVheJr8fZtbQzMaY2Vwz+9rMelWGz9vMLou+47PN7Dkzq1URP28zG2lmK81sdsKylH+++R1DKjl3r/A3wmyL3wF7AjWAmUCXuOMqwXm0ALpHj+sB3wBdgH8B10TLrwH+GT0+HngTMOAg4NNoeWNgQXTfKHrcKFr3WbStRa89Lu7zTjj/y4HRwOvR8xeBIdHjR4ALoscXAo9Ej4cAL0SPu0SffU2gXfSdqFqWvx/Af4Bzo8c1gIYV/fMGdge+B2onfM5nVsTPGzgc6A7MTliW8s83v2PoVrlvlaWEoCcw390XuPs24HlgQMwxFZu7L3P3z6PH64GvCf88BxAuHET3v4seDwBGefAJ0NDMWgC/Bd5x9zXu/iPwDtA3Wlff3T9xdwdGJewrVmbWCugHPB49N+AoYEy0Se7zznk/xgBHR9sPAJ53963u/j0wn/DdKJPfDzNrQLhgPAHg7tvcfS2V4PMmTM1e28yqAXWAZVTAz9vdPyBM/Z6oND7f/I4hlVhlSQh2BxYnPF8SLSu3omLR/YFPgebuvixatRxoHj3O77wLWr4kj+Vlwb3A1UB29LwJsNbdM6PnibH+fH7R+nXR9sV9P+LWDsgAnoyqSh43s12o4J+3u6cDdwI/EBKBdcB0Kv7nnaM0Pt/8jiGVWGVJCCoUM6sLvAz82d1/SlwX/RKoUH1JzewEYKW7T487llJWjVCc/LC77w9sJBTv/qyCft6NCL9g2wEtgV2AvrEGFZPS+Hwr4ndISqayJATpQOuE562iZeWOmVUnJAPPuvt/o8UrouJBovuV0fL8zrug5a3yWB63Q4D+ZraQULx7FHAfoci0WrRNYqw/n1+0vgGwmuK/H3FbAixx90+j52MICUJF/7yPAb539wx33w78l/AdqOifd47S+HzzO4ZUYpUlIZgKdIhaKdcgNDwaF3NMxRbViz4BfO3udyesGgfktCweBryasPyMqHXyQcC6qJhwAtDHzBpFv8b6ABOidT+Z2UHRsc5I2Fds3P1ad2/l7m0Jn91Edz8NmAT8Idos93nnvB9/iLb3aPmQqFV6O6ADodFVmfx+uPtyYLGZ7RUtOhqYQwX/vAlVBQeZWZ0orpzzrtCfd4LS+HzzO4ZUZnG3aiytG6GF7jeE1sV/jTueEp7DoYSivS+BGdHteEJ96f+Ab4F3gcbR9gY8FJ3zLCAtYV9nExpZzQfOSlieBsyOXvMg0WiWZeUG9OaXXgZ7Ev7BzwdeAmpGy2tFz+dH6/dMeP1fo3ObR0KL+rL6/QD2A6ZFn/krhFbkFf7zBm4C5kaxPU3oKVDhPm/gOUI7ie2EEqFzSuPzze8YulXum4YuFhERkUpTZSAiIiIFUEIgIiIiSghERERECYGIiIighEBERERQQiCCmWWZ2Qwzm2lmn5vZwYVs39DMLizCft8zs7QSxjTezBqW5LUiIiWhhEAENrv7fu7eDbgW+Ech2zckzLCXMu5+vIeJjERESoUSApEd1Qd+hDBnhJn9Lyo1mGVmOTPi3Q78JipVuCPa9i/RNjPN7PaE/Q0ys8/M7BszOyz3wcyshZl9EO1rds42ZrbQzJqa2fnRuhlm9r2ZTYrW9zGzKVFsL0XzW4iIlJgGJpJKz8yyCCO/1QJaAEe5+/ScqXfd/Sczawp8Qhj+dg/CaIldo9cfB/wNOMbdN5lZY3dfY2bvAdPd/QozOx643N2PyXXsK4Ba7n6rmVWNjrc+mrchzd1XRdtVByYS5rGfQhjf/zh332hmfyGM2ndzKt8nEanYqhW+iUiFt9nd9wMws17AKDPrShgq9jYzO5ww7fLu5D1N7DHAk+6+CcDdE+e3z5mAajrQNo/XTgVGRhf8V9x9Rj4x3kcYo/+1aPbHLsDHYYh6ahCSBBGRElNCIJLA3adEpQHNCOPdNwN6uPv26Fd7rWLucmt0n0Uef2/u/kGUcPQDnjKzu919VOI2ZnYmoVTi4pxFwDvufkoxYxERyZfaEIgkMLNOQFXC9LkNgJVRMnAk4aIMsB6ol/Cyd4CzzKxOtI/GxTjeHsAKdx8BPE6Y3jhxfQ/gSuB0d8+OFn8CHGJm7aNtdjGzjsU7UxGRHamEQARqm1lOUb0Bw9w9y8yeBV4zs1mEGQfnArj7ajP72MxmA2+6+1Vmth8wzcy2AeOB64p47N7AVWa2HdhAmKI20cVAY2BSVD0wzd3PjUoNnjOzmtF2/0eYvU9EpETUqFBERERUZSAiIiJKCERERAQlBCIiIoISAhEREUEJcN5hUAAAABdJREFUgYiIiKCEQERERFBCICIiIsD/AzjSbRzqavU2AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plot_absolute_time(results, [2], max_batch_size=max_batch_size)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "cf6a0dfc", + "metadata": { + "pycharm": { + "name": "#%%\n" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhsAAAEWCAYAAADPUVX+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd5gUVdbH8e8hgySBERFUUAliAkHW+KqorIhZQcwJcWVNGNacs655XRXFgIJZERWzsGaUARWQqGIgg5KVNOf9497RdpzQM0xPTfh9nqee6bqVTldXV5+5deuWuTsiIiIimVIt6QBERESkclOyISIiIhmlZENEREQySsmGiIiIZJSSDREREckoJRsiIiKSURU22TAzN7OtCpn+gJldkTJ+hpnNM7PlZta0BNtrHbdZo6Qxl5b4HrZIOo6yYmaPmdn1xZi/0GOjvDGz183sxKTjSIeZbRaPv+olWLaumb1iZkvM7LlMxCclY2bHmtlbScchlVeRyYaZzTSz1WbWLE/5+HhSbx3HH4vjh+SZ785YflIcP8nM1sUT1nIz+87MHjWzdqX2rgB3/4e7Xxe3WRO4A+jh7vXdfVEmf5Die/ywlNY12sz6pZbF9/Btaay/NJSHRMzMdjGzj5PafrrM7GozezK1zN17uvvjScVUmPj93zd33N1/iMffuhKs7kigOdDU3XuXQmy1zOz5GKOb2V55ppuZ3WJmi+Jwi5lZyvROZpZtZivj307rG1MSinu+ye/76u5D3b1HZiIUSb9m4zvg6NwRM9sOqJfPfNOAE1LmqwH0Ab7JM98n7l4faATsC/wKZJvZtumHXrB8/utqDtQBJpXG+qV0lVKS0gsYWQrrqTISSA43B6a5+9riLlhIrB8CxwFz85nWHzgU2AHYHjgIOD2urxbwMvAksCHwOPByLM+YmABV2BplkRJz90IHYCZwOfB5Stm/gcsAB1rHssdi+Txgw1h2IPA64YRwUiw7Cfgwn+28CjxfSBwXAnOA2cApcdtbpWz7fsKPzQpCAvMYcD3QLpY5sBx4D3g/jq+IZUfls73q8f0sBL4F/hmXqRGnNwIGx5hmxW1VB7YGfgPWxXUvjvPXjuv7Ie6jB4C6Kds7BPgCWEpIzvYHbojr+S2u6z9x3tT33ggYAiwAvo+fVbXUfR23+wshaexZxGd9CfB1nP9RoE6cNhE4KGXemnHfdI7vKXf/Lgd2ISSyl8eY5scYG8VlW8f5T43Lvh/Ldwc+BhYDP/LHMfMYcB/wGrAMGANsmSf2ccCO+eyf3eO69orjHYC3gZ+BqUCflHUUtS8/Av4DLAGmAPukLHsS4ThZFvfzsfns3/2B1cCauJ++jOWjgX55tnNn3A/fArvG8h/jvjwxZZ2FHld5tp+67kWEY3ZLwndiUfw8hwKN4/xPADmEfwaWA/9K+exyvwebACPi/pwBnFbAtq/J895PpQTHSCHH7k+5n3FK2cdA/5TxU4FP4+sehO+tpUz/Adi/gPWPBm4CPiN8R18GmqRM35k/jt0vU2OJy94Q9/2vwFax7Pq4zHLgFaBp3P9Lgc/549z6p32eesxQ8PmmFzA+rutH4Oo87zPv9/UkUs7LhGPuc8Kx/jmwa55tXxffzzLgLaBZnFaHkMAtivvic6B5Ub8zGir/UPQM4QdoX8KJeWvCD+pPhP9S8iYb1wODgDNi2bOEGpF0ko1TgHkFxLA/4US6LbABMIy/JhtLgN0IJ7A6ufHE6fl9WX9fvoBt/oPwg7Ip0AQYxZ9Psi8BD8Z4NiKchE4v6D0STvAj4roaEE4uN8Vp3WL8+8X4WwId4rTRxB+i/GInnKBfjutsTahdOjUljjXAafFzO4OQrFkB73kmIanIfc8fpezDfwHPpMx7CDChkP17CuHHZwugPvAi8ESe+YfE/VeXcDwti8dLTcKJt1PK57so7qcahBPy0ynbakHKD0fu/iEcNz8C3WL5BnH85LiezoQf2I5p7su1wMAY31HxM2sS17sUaJ8SzzYF7OOrgSfzlP3+Gads5+T4mV1P+HG4j5BY9Ij7qX5Rx1U+285d91nx/deN+2m/uO4sQiJ+V97vf8r4nz7rOP9/Cd+5ToRErXs6751iHiNFnKfySzaWAH9LGe8KLIuvBwKv55n/VeD8AtY/mnCM5Z6DXsh9L4Tv6yLgAML3d784npWy7A/ANnG/14xlMwjJXiNCgj+NcK6tEd/3o4V8v/IeM3nPN3sB28V4tiecPw8tZH2/r4NwLP0CHB9jOTqON03Z9jeEf+TqxvGb47TTCcdgPcLx2wVoWNTvjIbKPxSnOu8JwiWS/YDJhC9efoYAJ5hZY2BPYHia659NOMjz04fwxZvo7isIJ628Xnb3j9w9x91/S3ObhelDOOn+6O4/E/6rAcDMmhNOLOe6+wp3n0846ffNb0XxOnF/YKC7/+zuy4AbU+Y/FXjE3d+O8c9y9ylFBRgvF/UFLnH3Ze4+E7idcJLI9b27P+ThGvvjhB/C5oWs9j8p7/kG/rh89iRwgJk1jOPHE46JghwL3OHu37r7ckKNSd881eFXx/33K3AM8I67P+Xua9x9kbt/kTLvS+7+mYcq+KGEH7ZcBwBvuLunlPUmJIM93f2zWHYgMNPdH3X3te4+nvCj0TvNfTmfcEyscfdnCAl4rzgtB9jWzOq6+xx3X59Ldt/FGNcBzxCSv2vdfZW7v0WoIdgqjeMqP7Pd/d74/n919xnxuFvl7gsIbZv2TCdIM9uUkOBf5O6/xc/rYVIupRahuMdIcdUnJBy5lgD1437LOy13eoNC1vdEyjnoCqBPPG6OA0a6+8j4/X0bGEs4LnM95u6T4n5fE8sedfdv3H0JoQb4G3d/Jx7jzxGS4RJx99HuPiHG8xXwFGl+roRjerq7PxHjfYrwj9dBKfM86u7T4ufyLH98H9cQ/lHYyt3XuXu2uy8t6fuQyqM412yfIPwX04aQUOTL3T80syzCZZZX3f3XlDZZhWlJqIrNzyZAdsr49/nM82M6GymGTfKsM3WbmxP+O5mT8t6qFRJDFiHTz05tn0bI/CH8mJSkvUGzGEdqbN8T9mWu369lu/vKuP36hawz73veJC4728w+Ao4ws5eAnsA5haxnk3ziqsGfE53UbW3KX9v2pEq9Jr+SP7+HAwi1XanOBYa4+8SUss2Bv5nZ4pSyGoRjO519OStPQvM9sIm7rzCzo4ALgMFxP52fTsJYgHkpr38FcPe8ZfUp+rjKz5+O0Zg43w3sQfihrUb4LzYdmwC5SU6u7wk1COkuX5xjpLiWAw1TxhsCy93dzSzvtNzpyyhY3u9GTcJxszkhYU39Ma5JqA3Nb9lceT/T/D7jEjGzvwE3E2piahFqrtK9Ayjv5wKFnFf48/fxCcJ3+en4D+eTwGUpCZZUUWnXbLj794Rr0QcQqjsL8yRwPoUkJfk4DPiggGlzCAdwrs3yC7EY20pHYdv8EVhFuE7ZOA4N3X2bAmJZSDh5bJMyfyMPjWRz17dlAXEU9r4WEv6T2DxPnAXVOqUj73uenTL+OOG/uN6ERr6528kvxtn5xLWWP59QU5crbB8UKN5ptCehHUaq3sChZpaaEP0I/C/lM2js4c6KM0hvX7a0P2fOv+8fd3/T3fcj1BxNAR4qIOTSPE6LOq7S2f6NsWw7d29I+HytkPlTzQaamFlqbUBxjr/iHiPFNYnQODTXDvzRSHwSsH2ez3N7Cm9Enve7sYbwGfxIqPVIPa42cPebU+Zfn/exIv5NbZS/cRHrHka4vLapuzcitOWxQuZPlfdzgTQ/11jrd427dyS0+ziQ9Gu6pBIrbqvoUwnXY1cUMd89hMst7xc2k5lVN7M2ZnYv4RrjNQXM+ixwkpl1NLN6wFXFCztf8wjXigvyLHC2mbUysw2Bi3MnuPscQqOo282soZlVM7MtzSy3mnIe0Cq3Zbu75xB+fO40s40AzKylmf09zj8YONnM9onramlmHYqKM1azPwvcYGYNzGxz4DxCsldS/4zvuQmhduqZlGnDgR0JNRqpieQCwmWE1DifAgbGz7c+4UftGS/4ToShwL5m1sfMaphZ0zRvRdwd+CqfqtrZwD7AOWZ2Rix7FWhnZsebWc047GRmW6e5LzciHBM1zaw3oQ3TSDNrbmaHmNkGhCR0edwf+ZkHtC6NOxLSOK7S0YAQ7xIza0loiJ033oKOvx8JDRxvMrM6ZrY94RyR7vFX3GPkL8ystpnViaO1Yhy5P6pDgPPiPtmE8A/QY3HaaEKjyrPjOs6M5e8VsrnjUs5B1xIatK8jvN+DzOzv8ZxWx8z2MrNW6b6PwsTLW7Pi9qub2Sn8OTH/0/kmakCodfrNzLoRLlPmyu/7mmok4XtyTPwuHgV0JHx/CmVme5vZdvHy0lJCQlbQd0GqkGKd8OL1xbFpzPezu7+bp8o51S6xGnMp4UvfENjJ3ScUsL7XgbsIJ4IZFH5CSNfVwONmttjM+uQz/SHgTULL8nH8tTbnBEL1ZO6dG88T/qslxjcJmGtmC2PZRTH2T81sKfAO0D6+v88IDQLvJFw3/h9//GdxN3Ckmf1iZvfkE+dZhP98viU0xB0GPJLeLsjXMEIi9S3hssbvnWnF67MvEC6lvZhSvpLY2j7uz51jDLmX3r4jtJg/q6CNuvsPhFqz8wmX077gz/+VFqTAW17jOvcBLjazfrG6vwehTcNsQlXwLYQqZih6X44B2hL+m70BONLdFxG+R+fFdf5MqGk5g/zlVmUvMrNxaby/ohR4XKXpGkICuYRwt0/e4/wm4PL4uV6Qz/JHExocziY0mr7K3d9Jc9vFOkYKMJVQu9OS8H39lT++Ow8SGitOIDR8fi2W4e6rCbfFnkC4a+IUQgPK1YVs6wlCsjKX0CD27LiuHwkNpi8l/JD/SEjaSvMW19PiOhcRGpqm9imT3/lmAHCtmS0DriQk0sR48/u+kjJ9EaFG4vy4vX8BB7r7Qoq2MeFcuJTQtu9/FN62S6oIKzgfkKrGzGYSWrgX+GNhZlcC7dz9uDILrBBm9jXhR//rDG/nJMK+2T2T25HyycxGE+4+eTjpWEQqosS73paKI15aOZU/36GRmFhtPCTTiYaIiKwf9WQnaTGz0wjVw6+7e6FtccqKu6/O0whPRETKIV1GERERkYxSzYaIiIhkVIVos9GsWTNv3bp10mGIiFQo2dnZC909K+k4RCpEstG6dWvGji3yjlsREUlhZvn1tixS5nQZRURERDJKyYaIiIhklJINERERySglGyIiIpJRSjZEREQko5RsiIiISEYp2RAREZGMUrIhIlIO5eTA44/Ds88WPa9IeadkQ0SknPn0U9hlFzjpJBg6NOloRNafkg0RkXJi1iw44YSQaPz4Y6jZeOmlpKMSWX8VortyEZHK7Lff4I474MYbYc0auOSSMDRokHRkIqVDyYaISELc4cUX4YILYOZMOPxwuO022GKLpCMTKV26jCIikoCvvoLu3eHII6F+fXj3XXjhBSUaUjkp2RARKUMLF8IZZ0DnziHh+O9/Yfz4kHiIVFa6jCIiUgbWrAmJxdVXw7JlcOaZcNVV0KRJ0pGJZJ6SDRGRDHvzTTj3XJgyBfbbD+66Czp2TDoqkbKjyygiIhkybRocdBDsvz+sXQsjRoTEQ4mGVDVKNkREStmSJXDhhbDttvC//8Gtt8LEiSHxMEs6OpGyp8soIiKlZN06eOwxuPRSWLAATj4ZbrgBNt446chEkpXxmg0zq25m483s1Tj+mJl9Z2ZfxKFTpmMQEcm0Dz+Ebt2gXz/Yaiv47DMYPFiJhgiUzWWUc4DJecoudPdOcfiiDGIQEcmIH36Avn1hjz1g/nwYNiwkHl27Jh2ZSPmR0WTDzFoBvYCHM7kdEZGytnJluI21Qwd4+eVwG+uUKXD00WqXIZJXpms27gL+BeTkKb/BzL4yszvNrHaGYxARKTXu8PTTIcm45prQ6HPKlJB4bLBB0tGJlE8ZSzbM7EBgvrtn55l0CdAB2AloAlxUwPL9zWysmY1dsGBBpsIUEUlbdna4XHL00dC0abjT5JlnYPPNk45MpHzLZM3GbsDBZjYTeBrobmZPuvscD1YBjwLd8lvY3Qe5e1d375qVlZXBMEVECjdvXmj4udNOoe+MQYNg7Fj4v/9LOjKRiiFjyYa7X+Lurdy9NdAXeM/djzOzFgBmZsChwMRMxSAisj5Wr4Z//xvatYPHH4eBA0OycdppUL160tGJVBxJ9LMx1MyyAAO+AP6RQAwiIgVyh9deg/POg+nToVcvuP12aN8+6chEKqYySTbcfTQwOr7Wsw1FpNyaPDnUYLz5ZkguRo6Enj2TjkqkYlN35SIiwC+/hIelbbcdfPop3HknTJigREOkNKi7chGp0tatg4cegssvh59/hv794brrQO3SRUqPajZEpMoaNQp23BHOOCM8NG3cOHjgASUaIqVNyYaIVDnffQdHHgndu4cntD73XEg8OulJTSIZocsoIlJlLF8ON98cbmetXj1cLjn/fKhbN+nIRCo3JRsiUunl5IQHpF10EcyeDcceG5KOVq2SjkykatBlFBGp1D77DHbbDY4/HjbZBD76CJ58UomGSFlSsiEildLs2XDiifC3v8HMmfDYYzBmDOy6a9KRiVQ9uowiIpXKb7+FPjJuuAHWrIGLL4ZLL4UGDZKOTKTqUrIhIpWCOwwfHhp8fvcdHHpoaAi65ZZJRyYiuowiIhXehAmw775w+OFQrx68/Ta89JISDZHyQsmGiFRYixbBP/8Z+scYPx7uvRe++CIkHiJSfugyiohUOGvWhJ4+r7oKli6FAQPg6quhadOkIxOR/CjZEJEK5e23wwPTvv4a9tkH7rordDUuIuWXLqOISIUwYwYccgj06BHuOBk+PCQeSjREyj8lGyJSri1dGnr+7NgR3nsv9Pz59dch8TBLOjoRSUfGkw0zq25m483s1TjexszGmNkMM3vGzGplOgYRqXhycuDRR6FdO7j11tDF+LRpIfGoXTvp6ESkOMqiZuMcYHLK+C3Ane6+FfALcGoZxCAiFcjHH0O3bnDKKdCmTehy/NFHoUWLpCMTkZLIaLJhZq2AXsDDcdyA7sDzcZbHgUMzGYOIVBw//RRqMHbbDebMCc8w+fhj2GmnpCMTkfWR6ZqNu4B/ATlxvCmw2N3XxvGfgJb5LWhm/c1srJmNXbBgQYbDFJEk/fpreNx7+/bwwgtw+eUwdWpIPNQuQ6Tiy1iyYWYHAvPdPbsky7v7IHfv6u5ds7KySjk6ESkP3OG556BDB7jySjjgAJgyJSQe9esnHZ2IlJZM9rOxG3CwmR0A1AEaAncDjc2sRqzdaAXMymAMIlJOjR8P55wDH3wAO+wAQ4bAnnsmHZWIZELGajbc/RJ3b+XurYG+wHvufiwwCjgyznYi8HKmYhCR8mf+fOjfH7p0gcmT4cEHITtbiYZIZZZEPxsXAeeZ2QxCG47BCcQgImVs9Wq44w5o2zbcWXLuuTB9ekg8qldPOjoRyaQy6a7c3UcDo+Prb4FuZbFdESkfRo6EgQNDPxn77w933hnaaYhI1aAeREUkY6ZMCY0+e/UK46+9Bq+/rkRDpKpRsiEipW7xYjjvPNhuO/joI7j9dpgwISQeIlL16KmvIlJq1q2DwYPhsstg0SLo1w+uvx422ijpyEQkSarZEJFS8b//hTtMTj8dtt463GEyaJASDREpQbJhZtXMrGEmghGRiuf776FPH9hrL/j5Z3jmmZB4dO6cdGQiUl6klWyY2TAza2hmGwATga/N7MLMhiYi5dmKFaHXzw4d4NVX4ZprQoPQPn3UxbiI/Fm6NRsd3X0p4aFprwNtgOMzFpWIlFvuMGxYeI7JddfBYYeF55hceSXUq5d0dCJSHqWbbNQ0s5qEZGOEu68BPHNhiUh5NHYs7L57eEBa8+ahq/Fhw2DTTZOOTETKs3STjQeBmcAGwPtmtjmwNFNBiUj5MncunHJKeNT7jBnhjpPPPw+Jh4hIUdK69dXd7wHuSSn63sz2zkxIIlJerFoFd98dLpesWgUXXhge/95QTcRFpBjSSjbMrClwFbA74fLJh8C1wKLMhSYiSXGHESPg/PPhm2/goINCx1xt2yYdmYhUROleRnkaWAAcQXhi6wLgmUwFJSLJmTQJevSAQw+FWrXgzTdD4qFEQ0RKKt1ko4W7X+fu38XheqB5JgMTkbL1889w1lmwww6hIeg998CXX4bEQ0RkfaSbbLxlZn1jh17VzKwP8GYmAxORsrF2Ldx3X6i5+O9/Qw+g06eHxKNmzaSjE5HKIN1k4zRgGLA6Dk8Dp5vZMjPTXSkiFdS770KnTnDmmaFG44svQuLRrFnSkYlIZZJWsuHuDdy9mrvXiEO1WNbA3dUuXaSC+eab0BnXvvvCypXw4osh8dhuu6QjE5HKKO2nvprZwcD/xdHR7v5qEfPXAd4HasftPO/uV5nZY8CewJI460nu/kVxAxeR4lu2DG68Ee64I1wiufFGGDgQ6tRJOjIRqczSvfX1ZmAnYGgsOsfMdnP3SwpZbBXQ3d2Xx95HPzSz1+O0C939+RJHLSLFkpMDTzwBF18cOug64QS46SbYZJOkIxORqiDdmo0DgE7ungNgZo8D44ECkw13d2B5HK0ZB3VxLlLGPvkEzjkn9PjZrRsMHw5/+1vSUYlIVVKcR8w3TnndKJ0FzKy6mX0BzAfedvcxcdINZvaVmd1pZrULWLa/mY01s7ELFiwoRpgiAjBrFhx/POy6K/z0EwwZEhIPJRoiUtbSTTZuAsab2WOxViMbuKGohdx9nbt3AloB3cxsW0JtSAfCZZkmwEUFLDvI3bu6e9esrKw0wxSRX3+FG26Adu3guefg0kth2rSQeFQrzr8XIiKlJN1nozxlZqMJCQLARe4+N92NuPtiMxsF7O/u/47Fq8zsUeCC4gQsIvlzD3eVXHABzJwJhx8Ot90GW2yRdGQiUtWl9X+OmRmwD6Hdxgiglpl1K2KZLDNrHF/XBfYDpphZi5R1HgpMXI/4RYTQ02f37nDkkdCgQbiN9YUXlGiISPmQbqXqf4FdgKPj+DLgviKWaQGMMrOvgM8JbTZeBYaa2QRgAtAMuL7YUYsIAAsWwBlnwI47woQJoQfQceNC4iEiUl6kezfK39x9RzMbD+Duv5hZrcIWcPevgM75lOs0KLKe1qwJicXVV4e+M848E666Cpo0SToyEZG/SjfZWGNm1Ym3rppZFpCTsahEpEBvvBE64poyJTwk7c47oWPHpKMSESlYupdR7gFeAjYysxuADwl3qIhIGZk2DQ48EHr2DA9Pe+WVkHgo0RCR8i7du1GGmlk2oZGoAYe6++SMRiYiACxZAtddFx75XqdOuMPkrLOgdr491IiIlD/pdld+qrsPBqaklN3s7hdnLDKRKm7dOnj00dBPxsKFcMopof+M5s2TjkxEpHjSbbNxhJn95u5DAczsPkCPbhLJkA8+CF2Mjx8Pu+0Gr78OXbokHZWISMmknWwAI8wsB9gfWOzup2YuLJGq6Ycf4F//gmeegVat4Kmn4KijwCzpyERESq7QZMPMUm+k6wcMBz4CrjGzJu7+cyaDE6kqVq6EW2+FW24J41ddFZKOevWSjUtEpDQUVbORTbjd1VL+9oqDA+qfUGQ9uIdajAsvDA9LO+qokHRstlnSkYmIlJ5Ckw13b1NWgYhUNdnZoV3GRx9B584wbBjssUfSUYmIlD49A1KkjM2dC6eeCjvtFPrOeOgh+PxzJRoiUnml20BURNbT6tWhr4xrrw2PgT/vPLjiCmjUKOnIREQyq6gGojXdfU1ZBSNSGbnDa6+F5GL6dOjVC26/Hdq3TzoyEZGyUdRllE/MbLiZ/cPMWpdBPCKVyuTJoXvxgw6CatVg5Eh49VUlGiJStRSabLh7V+DcOHqXmX1uZneaWQ8zU2fJIgX45Rc491zYbjv49NPwsLQJE0LiISJS1RTZQNTdZ7r7A+5+KLAr8AqwL/CBmb2W6QBFKpK1a+H++6FtW7j3XujXL1w6OfdcqFkz6ehERJJRrAaisf3Ge3HAzFpmIiiRimjUqHAr64QJsOeecPfdsMMOSUclIpK89br11d1nFTbdzOqY2Wdm9qWZTTKza2J5GzMbY2YzzOwZM6u1PnGIJOm77+CII6B7d1i6FJ5/PiQeSjRERIJM97OxCuju7jsAnYD9zWxn4BbgTnffCvgF0HNWpMJZvhwuuwy23hreeAOuvz40CD3iCD3LREQkVUaTDQ+Wx9GacXCgO/B8LH8cODSTcYiUppwceOIJaNcObrwRevcOnXNddhnUrZt0dCIi5U9abTbMrB1wIbB56jLu3j2NZasTnrGyFXAf8A3hqbFr4yw/AX9p+2Fm/YH+AJvpQRFSTowZE9pljBkTegB94QXYZZekoxIRKd/SbSD6HPAA8BCwrjgbcPd1QCczawy8BHRIc7lBwCCArl27enG2KVLaZs+GSy6BIUNg443hscfg+OND3xkiIlK4dJONte5+//psyN0Xm9koYBegsZnViLUbrYBCG5qKJOW330IfGTfcAGvWwMUXw6WXQoMGSUcmIlJxFPp/mZk1MbMmwCtmNsDMWuSWxfJCmVlWrNHAzOoC+wGTgVHAkXG2E4GX1+tdiJQyd3jpJejYMSQX++0HX38NN92kRENEpLiKqtnIJjTozG1bf2HKNAe2KGL5FsDjsd1GNeBZd3/VzL4Gnjaz64HxwOBiRy6SIRMmhE643nsPttkG3n4b9t036ahERCquQpMNd2+zPit396+AzvmUfwt0W591i5S2hQvhyivhwQehcWO47z7o3x9q6NnIIiLrJa3mbWb2z9zLIXF8QzMbkLmwRMrOmjXh0e9t28KgQTBgQOhifMAAJRoiIqUh3bb0p7n74twRd/8FOC0zIYmUnbfeCj19nnMOdOkCX3wRnmnSpMgWSSIikq50k43qZn/0iRjbYKiLcamwZsyAQw6Bv/8dVq2C4cND24xtt006MhGRyifdZONN4Bkz28fM9gGeAt7IXFgimbF0KVx0UbjL5L334Oabw10mhxyiLsZFRDIl3SvS/yL05nlGHH8beDgjEYlkQE5O6DWuH9QAACAASURBVIjr0kth3jw4+eTQd0aLFklHJiJS+RWZbMRLJkPc/VhCL6IiFcpHH4U2GdnZoWvxV14JXY2LiEjZKPIySuxufHM9Bl4qmh9/hGOOgd13h7lzYejQkHgo0RARKVvpXkb5FvjIzEYAK3IL3f2OjEQlsh5WroR//zu0x3CHK64I7TQ22CDpyEREqqZ0k41v4lANUGfNUi65w3PPwYUXwg8/hEe/33ortG6ddGQiIlVbWsmGu18DYGb14/jyTAYlUlzjx4d2GR98EPrNGDIE9twz6ahERATS70F0WzMbD0wCJplZtpltk9nQRIo2f37oUrxLF5g8OXQ1np2tRENEpDxJt5+NQcB57r65u28OnA88lLmwRAq3ejXccUfoYvzRR8OD06ZPD4lH9epJRyciIqnSbbOxgbuPyh1x99FmpuZ2koiRI2HgQJg2DXr2DElHhw5JRyUiIgVJt2bjWzO7wsxax+Fywh0qImVmypSQXPTqFcZfey0kHko0RETKt3STjVOALOBF4AWgGXBypoISSfXbb3DBBbDddvDxx6EmY8IEOOCApCMTEZF0pHsZZV93Pzu1wMx6A88VtICZbQoMAZoDDgxy97vN7GrCE2MXxFkvdfeRxQ1cqobvvgu3sGZnQ79+oYvxjTZKOioRESmOdJONS/hrYpFfWaq1wPnuPs7MGgDZZvZ2nHanu/+7eKFKVfPaa3D88eG5Ji+/DAcfnHREIiJSEoUmG2bWEzgAaGlm96RMakhIJgrk7nOAOfH1MjObDLRcv3ClKli3Dq68Em68ETp1guefhy23TDoqEREpqaLabMwGxgK/Adkpwwjg7+luxMxaA52BMbHoTDP7ysweMbMNixmzVGLz5kGPHiHR6NcvtNFQoiEiUrEVWrPh7l8CX5rZL8Cr7p5T3A3EXkdfAM5196Vmdj9wHaEdx3XA7YQGqHmX6094rD2bbbZZcTcrFdCHH8JRR8HPP4e+M046KemIRESkNKR7N0ofYLqZ3Wpmad9oaGY1CYnGUHd/EcDd57n7upi4PAR0y29Zdx/k7l3dvWtWVla6m5QKyD3cYbLXXlCvHnz6qRINEZHKJK1kw92PI1wG+QZ4zMw+MbP+seFnvszMgMHA5NSnw5pZi5TZDgMmlihyqRSWLg13m5x/fmgAOnZseLaJiIhUHunWbODuS4HngaeBFoREYZyZnVXAIrsBxwPdzeyLOBwA3GpmE8zsK2BvYOB6vQOpsCZMgK5dYfjw8Ej4F16ARo2SjkpEREpbWre+mtnBhE68tiL0ndHN3eebWT3ga+DevMu4+4eA5bM69akhPP44nHEGNG4Mo0bBHnskHZGIiGRKuv1sHEHoG+P91EJ3X2lmp5Z+WFJZ/fYbnHUWPPww7L03DBsGG2+cdFQiIpJJaSUb7n6imW0cazgc+Nzd58Zp72YyQKk8vv0WjjwSxo+HSy6Ba6+FGummuyIiUmGl1WYj1l58BhwOHAl8amZ/uV1VpCCvvAJduoTux195JfSjoURDRKRqSPd0/y+gs7svAjCzpsDHwCOZCkwqh7Vr4Yor4OabYccdQ2+gbdokHZWIiJSldJONRcCylPFlsUykQHPnwtFHw+jR0L8/3H031KmTdFQiIlLW0k02ZgBjzOxlQpuNQ4CvzOw8gNR+NEQAPvgg9Aa6eHG48+SEE5KOSEREkpJusvFNHHK9HP8W2KmXVE3ucPvtcPHFsMUW8OabsN12SUclIiJJSvdulGsyHYhUfEuWhG7Ghw+HI46ARx6Bhg2TjkpERJKWdg+iIoX58stwt8mrr4bnnDz3nBINEREJlGzIenv0Udh5Z/j119AYdOBAsPz6jhURkSpJyYaU2K+/Qr9+cMopsOuuobOu3XZLOioRESlv0u3Uq52ZvWtmE+P49mZ2eWZDk/Lsm29CgjF4MFx2Gbz1Fmy0UdJRiYhIeZRuzcZDwCXAGgB3/wrom6mgpHx7+eXQPuP770Mbjeuvh+rVk45KRETKq3STjXru/lmesrWlHYyUb2vXwr/+BYceClttBePGQa9eSUclIiLlXbr9bCw0sy0JHXphZkcCczIWlZQ7c+ZA377w/vvwj3/AnXeqN1AREUlPusnGP4FBQAczmwV8BxyXsaikXPnf/0JvoMuWwRNPwHH65EVEpBjSuozi7t+6+75AFtDB3Xd395mFLWNmm5rZKDP72swmmdk5sbyJmb1tZtPj3w3X+11IRrjDLbdA9+7QuDGMGaNEQ0REii+tmg0zawycALQGaljsRMHdzy5ksbXA+e4+zswaANlm9jZwEvCuu99sZhcDFwMXlfgdSEYsXgwnnggjRkDv3uGukwbqnF5EREog3csoI4FPgQlATjoLuPscYrsOd19mZpOBloSHuO0VZ3scGI2SjXJl/Hg48kj44Qe46y44+2x10iUiIiWXbrJRx93PK+lGzKw10BkYAzSPiQjAXKB5Acv0B/oDbLbZZiXdtBTT4MHwz39Cs2ahMeguuyQdkYiIVHTp3vr6hJmdZmYtYpuLJmbWJJ0Fzaw+8AJwrrsvTZ3m7k68wyUvdx/k7l3dvWtWVlaaYUpJrVwZegLt1w/22CPUbijREBGR0pBusrEauA34BMiOw9iiFjKzmoREY6i7vxiL55lZizi9BTC/uEFL6ZoxIyQWjz4KV1wBb7wByu9ERKS0pHsZ5XxgK3dfmO6KLbQiHQxMdvc7UiaNAE4Ebo5/X053nVL6XnopPBa+Rg0YORJ69kw6IhERqWzSrdmYAaws5rp3A44HupvZF3E4gJBk7Gdm04F947iUsTVr4MIL4fDDoX370BuoEg0REcmEdGs2VgBfmNkoYFVuYWG3vrr7h0BB9zDsk3aEUupmzw69gX7wAQwYAHfcAbVrJx2ViIhUVukmG8PjIBXcqFFw9NGhN9ChQ+GYY5KOSEREKru0kg13fzzTgUhm5eSE3kAvvxzatoV334Vttkk6KhERqQoKTTbM7Fl372NmE8jnFlV33z5jkUmp+eUXOOGE8Dj4o46Chx5Sb6AiIlJ2iqrZOCf+PTDTgUhmjBsXegP96Se45x4480z1BioiImWr0LtRUnr6HODu36cOwIDMhycl5Q6DBsGuu8LataE30LPOUqIhIiJlL91bX/fLp0w3SpZTK1eGvjNOPx323DPUbuy8c9JRiYhIVVVUm40zCDUYW5jZVymTGgAfZTIwKZlp08Jlk4kT4aqrQo+g1asnHZWIiFRlRbXZGAa8DtxEeBR8rmXu/nPGopISeeEFOPlkqFULXn8d/v73pCMSEREpItlw9yXAEuDosglHSmLNGrjoIrjzTvjb3+DZZ0EPyhURkfIi3U69pJyaNSvczvrRR+FOk9tvDzUbIiIi5YWSjQrsvfdCb6ArVsBTT4UuyEVERMqbdO9GkXIkJwduvBH22w+aNoXPP1eiISIi5ZdqNiqYn38OvYG+9lqo1Rg0COrXTzoqERGRginZqEDGjoXevUM7jf/8JzyxVZ10iYhIeafLKBWAOzzwAOy2G6xbFx4N/89/KtEQEZGKQclGObdiRbhscsYZ0L07jB8fbm8VERGpKDKabJjZI2Y238wmppRdbWazzOyLOByQyRgqsqlTQ2IxdChcc01op9G0adJRiYiIFE+mazYeA/bPp/xOd+8Uh5EZjqFCeu456NoV5s6FN96AK6+EaqqHEhGRCiijP1/u/j6gbs2LYfVqOPdc6NMHtt02XDbp0SPpqEREREouqf+VzzSzr+Jllg3zm8HM+pvZWDMbu2DBgrKOLxE//QR77QV33w3nnAP/+x9sumnSUYmIiKyfJJKN+4EtgU7AHOD2/GZy90Hu3tXdu2ZlZZVlfIl45x3o3BkmTIBnnoG77lK34yIiUjmUebLh7vPcfZ275wAPAd3KOobyJCcHrrsuXCpp3jz0BtqnT9JRiYiIlJ4y79TLzFq4+5w4ehgwsbD5K7NFi+D448Pj4I89Fh58EDbYIOmoRERESldGkw0zewrYC2hmZj8BVwF7mVknwIGZwOmZjKG8+vxzOPLIcLfJ/ffD6aerky4REamcMppsuPvR+RQPzuQ2y7vc3kDPPRdatIAPP4Sddko6KhGRzMnOzt6oRo0aDwPbos4kK6scYOLatWv7denSZX7eiXo2ShlasSLUYAwdCj17whNPqJMuEan8atSo8fDGG2+8dVZW1i/VqlXzpOOR0peTk2MLFizoOHfu3IeBg/NOV4ZZRqZMgW7dYNiw0CD01VeVaIhIlbFtVlbWUiUalVe1atU8KytrCaH26i9Us1EGnnkGTj0V6tWDt96CffdNOiIRkTJVTYlG5Rc/43wrMVSzkUGrV8PZZ0PfvrDDDjBunBINERGpepRsZMiPP8L//R/ce29oDDp6NLRqlXRUIiKSjosvvnjj0lrXeeedt8mVV17ZvKDpxxxzzGZvvfXWBt26dWv//vvv1yut7Zam+++/v0m7du06tmvXrmPnzp07fPLJJ3WLs7ySjQx4663QG+ikSfDss3DnnVCzZtJRiYhIuu65554WxZk/JyeHdevWlWhb48aNq9+9e/cVJVq4jGy11VarPvroo6nTpk37+pJLLpl9+umnb16c5ZVslKKcnPAo+P33h403hrFjoXfvpKMSEanapk6dWqtNmzbbHHzwwW222GKLbfbff/8tli1bVm3EiBEN9t133y1z53vppZca7rffflsOGDCg5apVq6p16NCh48EHH9wG4Oqrr27etm3bbdq2bbvNtddeu1Huelu3br3tYYcd1rpdu3bbfPPNN7Wef/75hh07dty6ffv2HXfZZZd2ueuePHly3W7durVv1arVdtdff/1GueXjxo2rs8UWW/xWo8YfTSjXrVvHEUcc0frss8/eBODFF19s2KlTpw4dO3bcumfPnlssWbKkGsDLL7/cYOutt+7Yrl27jr17927966+/GkDLli23+8c//tGqXbt2HbfbbrutJ06cWBvgkUce2bBt27bbtG/fvmPXrl3bF2cf7rfffiuysrLWAey9994r5s6dW6wHaqiBaClZuBCOOw7efDP0Cnr//eoNVEQkr1NOYdOJEynVSwXbbsvKRx7hx8LmmTlzZp0HH3xwZo8ePVb07t279W233ZZ19dVXzzvnnHM2mz17do1NNtlk7SOPPNL05JNPXnjMMccseeyxxzaaMmXK1wAffPBBvWHDhjXNzs6e7O506dJl63322WdZs2bN1v3www+1Bw8e/N0+++wzc/bs2TXOPPPM1qNHj57SoUOH1fPmzaueu/0ZM2bU+fjjj6cuXry4+tZbb73thRdeuKB27do+YsSIRj169FiSO9+aNWvs0EMPbdOxY8dfb7nllrlz5sypceONN7Z4//33pzVs2DDnsssu2/i6665rfu211849/fTT27z11ltTt99++1WHHXZY69tuuy3ryiuvnA/QqFGjtdOmTfv6P//5T9Ozzjpr01GjRs24+eabW7z11lvT2rRps2bhwoXV/7qX0nPvvfc223vvvZcUPecfVLNRCsaMgR13hFGjQoddjz+uRENEpDzZeOONV/fo0WMFwPHHH7/o448/rl+tWjX69Omz6KGHHmqycOHC6uPGjavfu3fvv/yIjh49uv4BBxywuGHDhjmNGjXK6dWr1y+jRo1qANCiRYvV++yzz4o43wbdunVb1qFDh9UAzZs3//26So8ePRbXrVvXW7RosbZJkyZrfvrppxoA77zzTsNDDz10ae58AwYM2Dw30chd5zfffFOnW7duHTp06NDx6aefbvrDDz/U+vLLL+u0atVq1fbbb78K4KSTTlr04YcfNshdz4knnvgzwGmnnfbz+PHj6wN07dp1+bHHHtv69ttvb7Z27doS7cdXXnmlwZNPPtns7rvv/qk4y6lmYz24w333wXnnQcuW8PHH0KVL0lGJiJRfRdVAZIrleR5E7vgZZ5yxqFevXlvVqVPHDzrooF9qFrOBXb169XLSma927dq/3/pbvXp11q5da8uWLau2dOnS6q1bt16TO61r167LP/jgg4YrV66cV69ePXd3dt9996WvvPLKd6nrK6qBZrVqf9QlmJkDDBs27If33ntvgxEjRjTq0qVLx+zs7K833njj3xOis846q+Xbb7/dCCC3VifVmDFj6g4YMGDz1157bXrqculQzUYJLV8OxxwDZ50Vntiana1EQ0SkvJozZ06td955ZwOAoUOHNtl1112XA7Ru3XpN8+bN19x+++0t+vfvvzB3/ho1aviqVasMYO+9914+cuTIxjE5qDZy5MgN995772V5t7HXXnut+OyzzxpMmTKlFkDqZZT8vPbaaw123333P63n9NNPX9ijR48lBx544JZr1qxhr732WjF27Nj6ue0uli5dWu2rr76qvcMOO/w2a9asWrnlQ4YMabrHHnv8vq4hQ4Y0ARg8ePCGnTt3XgEwadKk2t27d19x1113zd5www3Xfvvtt39qd3HvvffOmjJlytf5JRrTp0+v1bt37y0feeSR73JrU4pDNRslMHkyHHEETJ0KN94IF10E1ZS2iYiUW61bt/7t3nvv3ah///712rZt+9sFF1ywIHda3759F9133301dtxxx99yy4499tgFW2+9dcdtt9125YgRI7475phjFu24445bAxx//PELdtttt1+nTp36px/rTTbZZO0999wz87DDDtsqJyeHpk2brvn444+nFxTTyJEjG/Xp0+eXvOVXX331vIEDB1Y//PDD2wwfPvy7Bx98cGbfvn23WL16tQFcddVVs7bffvtVDzzwwMzevXtvuW7dOnbYYYeVqe/pl19+qd6uXbuOtWrV8qeffvpbgIEDB7aaOXNmbXe33XfffenOO+/8a7r77/LLL2+xePHiGmedddbmEJKxiRMnTk53eXMv/526de3a1ceOHZt0GAA89RScdlpok/HUU9C9e9IRiYjkz8yy3b1r0nF8+eWXM3fYYYeFRc+ZGVOnTq114IEHtp0+ffqk/KafcMIJm3Xu3HnlwIEDyzTGjh07bj1+/PgpqZdYSkPLli23Gzt27OQWLVqUrGHGevjyyy+b7bDDDq3zlqtmI02rVsH554c2GrvtFrogb9ky6ahERGR9bLPNNlvXrVs358EHHyzztiRff/112jUDFZ2SjTR8/z306QOffRYag958szrpEhGpKNq3b7+6oFqNSZMmVbof/FmzZk1IOoa8lGwU4Y034NhjYc0aeOEFOPzwpCMSEalwcnJyckwPY6vccnJyDMj37pyMNms0s0fMbL6ZTUwpa2Jmb5vZ9Ph3w0zGUFLr1sFVV8EBB4TLJdnZSjREREpo4oIFCxrFHyOphHJycmzBggWNgIn5Tc90zcZjwH+AISllFwPvuvvNZnZxHL8ow3EUy4IFoTbj7bfhxBPhv/8Nj4cXEZHiW7t2bb+5c+c+PHfu3G1RlwuVVQ4wce3atf3ym5jRZMPd3zez1nmKDwH2iq8fB0ZTjpKNTz4J7TMWLIBBg6BfPzDl4iIiJdalS5f5wMFJxyHJSSLDbO7uc+LruUC+j901s/5mNtbMxi5YsCC/WUqVO9xzT3gsfM2aoTfQ005ToiEiIrK+Eq3O8tDJR74Nhtx9kLt3dfeuWVlZGY1j2TLo2xfOOQd69gztM3bcMaObFBERqTKSSDbmmVkLgPh3fgIx/G7SJNhpJ3j+ebjpJhg+HDYsl01WRUREKqYkko0RwInx9YnAywnEAMDQodCtG/zyC7zzDlx8sbodFxERKW2ZvvX1KeAToL2Z/WRmpwI3A/uZ2XRg3zheplatggED4LjjwsPTxo+Hvfcu6yhERESqhkzfjXJ0AZP2yeR2C/P999C7N3z+OVx4YXiQWg11bSYiIpIxVepnduTIUJuxbh289BIcemjSEYmIiFR+VaKFwrp1cMUV0KsXbLZZuNtEiYaIiEjZqPQ1G/PnwzHHwLvvwsknh6e21q2bdFQiIiJVR6VONiZMCP1mLFoEgwfDKackHZGIiEjVU6kvo2y6KWyzTeiCXImGiIhIMip1zUbjxvDmm0lHISIiUrVV6poNERERSZ6SDREREckoJRsiIiKSUUo2REREJKOUbIiIiEhGKdkQERGRjFKyISIiIhmlZENEREQyytw96RiKZGYLgO/XYxXNgIWlFE5pUlzFo7iKR3EVT2WMa3N3zyrNYERKokIkG+vLzMa6e9ek48hLcRWP4ioexVU8ikskc3QZRURERDJKyYaIiIhkVFVJNgYlHUABFFfxKK7iUVzFo7hEMqRKtNkQERGR5FSVmg0RERFJiJINERERyagKmWyY2aZmNsrMvjazSWZ2TixvYmZvm9n0+HfDWG5mdo+ZzTCzr8xsx5R1nRjnn25mJ65nXHXM7DMz+zLGdU0sb2NmY+L2nzGzWrG8dhyfEae3TlnXJbF8qpn9fX3iSllndTMbb2avlpe4zGymmU0wsy/MbGwsS/RzjOtrbGbPm9kUM5tsZrskHZeZtY/7KXdYambnJh1XXN/AeMxPNLOn4nehPBxf58SYJpnZubGszPeXmT1iZvPNbGJKWanFYWZd4vdoRlzWSrbHRDLE3SvcALQAdoyvGwDTgI7ArcDFsfxi4Jb4+gDgdcCAnYExsbwJ8G38u2F8veF6xGVA/fi6JjAmbu9ZoG8sfwA4I74eADwQX/cFnomvOwJfArWBNsA3QPVS2G/nAcOAV+N44nEBM4FmecoS/RzjOh8H+sXXtYDG5SGulPiqA3OBzZOOC2gJfAfUTTmuTkr6+AK2BSYC9YAawDvAVknsL+D/gB2BiZk4zoHP4rwWl+1ZGseZBg2lNSQeQKm8CXgZ2A+YCrSIZS2AqfH1g8DRKfNPjdOPBh5MKf/TfOsZUz1gHPA3Qu9/NWL5LsCb8fWbwC7xdY04nwGXAJekrOv3+dYjnlbAu0B34NW4nfIQ10z+mmwk+jkCjQg/nlae4soTSw/go/IQFyHZ+JHwI1gjHl9/T/r4AnoDg1PGrwD+ldT+Alrz52SjVOKI06aklP9pPg0aysNQIS+jpIpVsJ0JtQjN3X1OnDQXaB5f554Mc/0UywoqX594qpvZF8B84G3Cf2eL3X1tPtv4fftx+hKgaSbiAu4inGhz4njTchKXA2+ZWbaZ9Y9lSX+ObYAFwKMWLjs9bGYblIO4UvUFnoqvE43L3WcB/wZ+AOYQjpdskj++JgJ7mFlTM6tHqDHYlPLzOZZWHC3j69KOT6TUVOhkw8zqAy8A57r70tRp7u6EH7Iy5e7r3L0ToSahG9ChrGPIy8wOBOa7e3bSseRjd3ffEegJ/NPM/i91YkKfYw1Clff97t4ZWEGo5k46LgBi24eDgefyTksirtjW4BBCkrYJsAGwf1nGkB93nwzcArwFvAF8AazLM09in2N5jEMkUypssmFmNQmJxlB3fzEWzzOzFnF6C0LtAsAswn80uVrFsoLK15u7LwZGEaqPG5tZjXy28fv24/RGwKIMxLUbcLCZzQSeJlxKubscxJX7XzHuPh94iZCgJf05/gT85O5j4vjzhOQj6bhy9QTGufu8OJ50XPsC37n7AndfA7xIOObKw/E12N27uPv/Ab8Q2nclvb9ylVYcs+Lr0o5PpNRUyGQjtrQeDEx29ztSJo0Aclton0hoy5FbfkJs5b0zsCRWX74J9DCzDeN/Zz1iWUnjyjKzxvF1XUI7ksmEpOPIAuLKjfdI4L34H84IoG9std8GaEtoAFYi7n6Ju7dy99aE6vf33P3YpOMysw3MrEHua8L+n0jCn6O7zwV+NLP2sWgf4Ouk40pxNH9cQsndfpJx/QDsbGb14nczd38lenwBmNlG8e9mwOGEBtJJ769cpRJHnLbUzHaO+/+ElHWJlA9JNxopyQDsTqhy/IpQNfoF4XpsU0IjyOmEludN4vwG3EdoPzEB6JqyrlOAGXE4eT3j2h4YH+OaCFwZy7cgnDRnEKq+a8fyOnF8Rpy+Rcq6LovxTqUUW5YDe/HH3SiJxhW3/2UcJgGXxfJEP8e4vk7A2PhZDie0/i8PcW1AqAVolFJWHuK6BpgSj/snCHeUJH7cAx8QEp8vgX2S2l+E5HAOsIZQc3ZqacYBdI37/hvgP+Rp3KxBQ9KDuisXERGRjKqQl1FERESk4lCyISIiIhmlZENEREQySsmGiIiIZJSSDREREckoJRtSqZnZOgtPSP3SzMaZ2a5FzN/YzAaksd7RZta1hDGNzO2PRUSkKlCyIZXdr+7eyd13IDzo66Yi5m9MeCppxrj7AR56mBURqRKUbEhV0pDQZTVmVt/M3o21HRPM7JA4z83AlrE25LY470Vxni/N7OaU9fU2s8/MbJqZ7ZF3Y2bWwszej+uamDuPmc00s2Zm9o847Qsz+87MRsXpPczskxjbcxaeASQiUmGpUy+p1MxsHaEXxjqER3F3d/fs+EyOeu6+1MyaAZ8SusfenNDD6rZx+Z6ER5Pv6+4rzayJu/9sZqOBbHc/38wOAM5z933zbPt8oI6732Bm1eP2lll4Rk1Xd18Y56sJvAfcCnxCeLZIT3dfYWYXEXrevDaT+0lEJJNqFD2LSIX2q4en8GJmuwBDzGxbQpfQN1p4ymwO4ZHczfNZfl/gUXdfCeDuP6dMy30AYDbQOp9lPwceicnEcHf/ooAY7yY8H+QVC0/o7Qh8FB5zQS1CAiIiUmEp2ZAqw90/ibUYWYRn6WQBXdx9TaxtqFPMVa6Kf9eRz3fJ3d+PyUwv4DEzu8Pdh6TOY2YnEWpTzswtAt5296OLGYuISLmlNhtSZZhZB6A68UFmwPyYaOxN+MEHWAY0SFnsbeBkM6sX19GkGNvbHJjn7g8BDxMeU586vQtwAXCcu+fE4k+B3cxsqzjPBmbWrnjvVESkfFHNhlR2dc0s9/KFASe6+zozGwq8YmYTCE93nQLg7ovM7CMzmwi87u4XmlknYKyZrQZGApemvNaUIQAAAGtJREFUue29gAvNbA2wnPDo71RnAk2AUfGSyVh37xdrO54ys9pxvsuBacV+5yIi5YQaiIqIiEhG6TKKiIiIZJSSDREREckoJRsiIiKSUUo2REREJKOUbIiIiEhGKdkQERGRjFKyISIiIhn1/2p/esEbfqUvAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plot_relative_time(results, [2], max_batch_size=max_batch_size)" + ] + }, + { + "cell_type": "markdown", + "id": "f7dc206c", + "metadata": { + "pycharm": { + "name": "#%% md\n" + } + }, + "source": [ + "## Conclusion\n", + "\n", + "As illustrated in the experiments, KeOps allows you to drastically speed up and scale up drift detection to larger datasets without running into memory issues. The speed benefit of KeOps over the PyTorch (or TensorFlow) MMD detector decreases as the number of features increases. Note though that it is not advised to apply the (untrained) MMD detector to very high-dimensional data in the first place." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/doc/source/overview/getting_started.md b/doc/source/overview/getting_started.md index 922a17543..9e987686f 100644 --- a/doc/source/overview/getting_started.md +++ b/doc/source/overview/getting_started.md @@ -155,6 +155,32 @@ The TensorFlow installation is required to use the following detectors: ``` ```` +````{tab-item} KeOps +:sync: label-keops +:class-label: sd-pt-0 + +```{div} sd-mb-1 +Installation with [KeOps](https://www.kernel-operations.io) backend. +``` + +```bash +pip install alibi-detect[keops] +``` + +```{div} sd-mb-1 +The KeOps installation is required to use the KeOps backend for the following detectors: +- [MMDDrift](../cd/methods/mmddrift.ipynb) +``` + +```{note} +KeOps requires a C++ compiler compatible with `std=c++11`, for example `g++ >=7` or `clang++ >=8`, and a +[Cuda toolkit](https://developer.nvidia.com/cuda-toolkit) installation. For more detailed version requirements +and testing instructions for KeOps, see the +[KeOps docs](https://www.kernel-operations.io/keops/python/installation.html). **Currently, the KeOps backend is +only officially supported on Linux.** +``` +```` + ````{tab-item} Prophet :class-label: sd-pt-0 @@ -199,9 +225,10 @@ mamba install -c conda-forge alibi-detect [Alibi Detect](https://github.com/SeldonIO/alibi-detect) is an open source Python library focused on **outlier**, **adversarial** and **drift** detection. The package aims to cover both -online and offline detectors for tabular data, text, images and time series. -Both **TensorFlow** and **PyTorch** backends are supported for drift detection. Alibi-Detect does not install these as -default. See [installation options](#installation) for more details. +online and offline detectors for tabular data, text, images and time series. **TensorFlow**, **PyTorch** +and (where applicable) [KeOps](https://www.kernel-operations.io/keops/index.html) backends are supported +for drift detection. Alibi-Detect does not install these as default. See [installation options](#installation) +for more details. To get a list of respectively the latest outlier, adversarial and drift detection algorithms, you can type: diff --git a/examples/cd_mmd_keops.ipynb b/examples/cd_mmd_keops.ipynb new file mode 120000 index 000000000..fddcc9f46 --- /dev/null +++ b/examples/cd_mmd_keops.ipynb @@ -0,0 +1 @@ +../doc/source/examples/cd_mmd_keops.ipynb \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 67491ae8d..5b4de7f3c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,6 +44,7 @@ envlist= tensorflow torch prophet + keops all # tox test environment for generating licenses @@ -113,6 +114,17 @@ extras= commands = {env:COMMAND:pytest --no-cov alibi_detect/tests/test_dep_management.py --opt-dep=prophet} +# tox test environment for testing keops optional dependency imports +[testenv:keops] +basepython = python +deps = pytest + pytest-cov + pytest-randomly +extras= + keops +commands = + {env:COMMAND:pytest --no-cov alibi_detect/tests/test_dep_management.py --opt-dep=keops} + # environment for testing imports with all optional dependencies installed [testenv:all] basepython = python diff --git a/setup.py b/setup.py index c56e392a0..f7dc86e60 100644 --- a/setup.py +++ b/setup.py @@ -23,12 +23,17 @@ def readme(): "tensorflow_probability>=0.8.0, <0.18.0", "tensorflow>=2.2.0, !=2.6.0, !=2.6.1, <2.10.0", # https://github.com/SeldonIO/alibi-detect/issues/375 and 387 ], - 'all': [ + "keops": [ + "pykeops>=2.0.0, <2.2.0", + "torch>=1.7.0, <1.13.0" + ], + "all": [ "fbprophet>=0.5, <0.7", "holidays==0.9.11", "pystan<3.0", "tensorflow_probability>=0.8.0, <0.18.0", "tensorflow>=2.2.0, !=2.6.0, !=2.6.1, <2.10.0", # https://github.com/SeldonIO/alibi-detect/issues/375 and 387 + "pykeops>=2.0.0, <2.2.0", "torch>=1.7.0, <1.13.0" ], } diff --git a/testing/test_notebooks.py b/testing/test_notebooks.py index 48a94c264..d885f4c9c 100644 --- a/testing/test_notebooks.py +++ b/testing/test_notebooks.py @@ -38,6 +38,7 @@ 'cd_context_20newsgroup.ipynb', 'cd_context_ecg.ipynb', 'cd_text_imdb.ipynb', + 'cd_mmd_keops.ipynb', # the following requires a k8s cluster 'alibi_detect_deploy.ipynb', # the following require downloading large datasets From 0bbe586ff4ccce76795d01f9cde7940b205ba18e Mon Sep 17 00:00:00 2001 From: arnaudvl Date: Mon, 22 Aug 2022 11:41:51 +0100 Subject: [PATCH 03/35] update changelog (#593) --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2918c71c..b5726153e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ ## v0.10.4dev [Full Changelog](https://github.com/SeldonIO/alibi-detect/compare/v0.10.3...master) +### Added +- **New feature** MMD drift detector has been extended with a [KeOps](https://www.kernel-operations.io/keops/index.html) backend to scale and speed up the detector. +See the [documentation](https://docs.seldon.io/projects/alibi-detect/en/latest/cd/methods/mmddrift.html) and [example notebook](https://docs.seldon.io/projects/alibi-detect/en/latest/examples/cd_mmd_keops.html) for more info ([#548](https://github.com/SeldonIO/alibi-detect/pull/548)). + ## v0.10.3 ## [v0.10.3](https://github.com/SeldonIO/alibi-detect/tree/v0.10.3) (2022-08-17) [Full Changelog](https://github.com/SeldonIO/alibi-detect/compare/v0.10.2...v0.10.3) From 586ceeade5385589bb83530df566ea788e23d18b Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Tue, 6 Sep 2022 09:19:19 +0100 Subject: [PATCH 04/35] Update CONTRIBUTING.md (#595) Expand guidance on docstrings for classes. --- CONTRIBUTING.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 28d15a0fc..7c8c3e7d0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,8 +36,14 @@ options are defined in `setup.cfg`. ## Docstrings We adhere to the `numpy` style docstrings (https://numpydoc.readthedocs.io/en/stable/format.html) with the exception of ommiting argument types in docstrings in favour of type hints in function -and class signatures. If you're using a `PyCharm`, you can configure this under -`File -> Settings -> Tools -> Python Integrated Tools -> Docstrings`. +and class signatures. If you use an IDE, you may be able to configure it to assist you with writing +docstrings in the correct format. For `PyCharm`, you can configure this under +`File -> Settings -> Tools -> Python Integrated Tools -> Docstrings`. For `Visual Studio Code`, you can obtain +docstring generator extensions from the [VisualStudio Marketplace](https://marketplace.visualstudio.com/). + +When documenting Python classes, we adhere to the convention of including docstrings in their `__init__` method, +rather than as a class level docstring. Docstrings should only be included at the class-level if a class does +not posess an `__init__` method, for example because it is a static class. ## Building documentation We use `sphinx` for building documentation. You can call `make build_docs` from the project root, @@ -104,4 +110,4 @@ replaced with an instance of the MissingDependency class. For example: ... ``` - Developers can use `make repl tox-env=` to run a python REPL with the specified optional dependency -installed. This is to allow manual testing. \ No newline at end of file +installed. This is to allow manual testing. From b25819d7aa5dd5ded43dbf18182ca0d79af69881 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Tue, 6 Sep 2022 15:43:57 +0100 Subject: [PATCH 05/35] Enforce utf-8 when reading README.md in setup.py (#605) Addresses #600. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f7dc86e60..4907088d8 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ def readme(): - with open("README.md") as f: + with open("README.md", encoding="utf-8") as f: return f.read() From ada0c66011a4b1463db2e3d7ad007d8450eb5275 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Tue, 6 Sep 2022 16:36:27 +0100 Subject: [PATCH 06/35] Feature/tabular warning (#606) --- alibi_detect/cd/tabular.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/alibi_detect/cd/tabular.py b/alibi_detect/cd/tabular.py index 94730771a..941de2e02 100644 --- a/alibi_detect/cd/tabular.py +++ b/alibi_detect/cd/tabular.py @@ -3,6 +3,7 @@ from typing import Callable, Dict, List, Optional, Tuple, Union from alibi_detect.cd.base import BaseUnivariateDrift from alibi_detect.utils.warnings import deprecated_alias +import warnings class TabularDrift(BaseUnivariateDrift): @@ -88,7 +89,7 @@ def __init__( self._set_config(locals()) self.alternative = alternative - self.x_ref_categories, self.cat_vars = {}, [] # no categorical features assumed present + # Parse categories_per_feature dict if isinstance(categories_per_feature, dict): vals = list(categories_per_feature.values()) int_types = (int, np.int16, np.int32, np.int64) @@ -106,6 +107,11 @@ def __init__( 'Dict[int, NoneType], Dict[int, int], Dict[int, List[int]]') self.x_ref_categories = categories_per_feature self.cat_vars = list(self.x_ref_categories.keys()) + # No categories_per_feature dict so assume no categorical features present + else: + self.x_ref_categories, self.cat_vars = {}, [] + warnings.warn('No `categories_per_feature` dict provided so all features are assumed to be numerical. ' + '`KSDrift` will be applied to all features.') def feature_score(self, x_ref: np.ndarray, x: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: """ From 571d5015de17f0063e4ad39fa042dd799819b610 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Tue, 6 Sep 2022 17:05:54 +0100 Subject: [PATCH 07/35] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b5726153e..d913dd9b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ ### Added - **New feature** MMD drift detector has been extended with a [KeOps](https://www.kernel-operations.io/keops/index.html) backend to scale and speed up the detector. See the [documentation](https://docs.seldon.io/projects/alibi-detect/en/latest/cd/methods/mmddrift.html) and [example notebook](https://docs.seldon.io/projects/alibi-detect/en/latest/examples/cd_mmd_keops.html) for more info ([#548](https://github.com/SeldonIO/alibi-detect/pull/548)). +- If a `categories_per_feature` dictionary is not passed to `TabularDrift`, a warning is now raised to inform the user that all features are assumed to be numerical ([#606](https://github.com/SeldonIO/alibi-detect/pull/606)). + +### Development +- UTF-8 decoding is enforced when `README.md` is opened by `setup.py`. This is to prevent pip install errors on systems with `PYTHONIOENCODING` set to use other encoders ([#605](https://github.com/SeldonIO/alibi-detect/pull/605)). ## v0.10.3 ## [v0.10.3](https://github.com/SeldonIO/alibi-detect/tree/v0.10.3) (2022-08-17) From 271158023e5ca5c2589d290fb738398f794ec84b Mon Sep 17 00:00:00 2001 From: arnaudvl Date: Thu, 8 Sep 2022 18:54:06 +0100 Subject: [PATCH 08/35] Learned kernel MMD with KeOps backend (#602) * first commit adding learned kernel mmd with keops backend * update method docs learned kernel * update preprocessing and types * add test and update output type deep kernel * update example * test equivalence learned kernel mmd2 keops with torch implementation * add deep kernel keops test and add skipif for all keops tests * remove print statement * fix flake8 * handle optional keops dependency * clarify bandwidth setting * handle keops optional dependency in test * update pydantic model schema * add DeepKernel to keops dependency management test * add keops to top level learned kernel test * update test learned kernel * clarify test variable and make proj type explicit * remove unnecessary metadata --- alibi_detect/cd/keops/learned_kernel.py | 341 ++++++++++++++ .../keops/tests/test_learned_kernel_keops.py | 130 ++++++ alibi_detect/cd/learned_kernel.py | 35 +- alibi_detect/cd/tests/test_learned_kernel.py | 22 +- alibi_detect/saving/schemas.py | 4 + alibi_detect/tests/test_dep_management.py | 1 + alibi_detect/utils/keops/__init__.py | 8 +- alibi_detect/utils/keops/kernels.py | 72 ++- .../utils/keops/tests/test_kernels_keops.py | 61 ++- .../cd/methods/learnedkerneldrift.ipynb | 51 ++- doc/source/examples/cd_mmd_keops.ipynb | 432 ++++++++++-------- 11 files changed, 933 insertions(+), 224 deletions(-) create mode 100644 alibi_detect/cd/keops/learned_kernel.py create mode 100644 alibi_detect/cd/keops/tests/test_learned_kernel_keops.py diff --git a/alibi_detect/cd/keops/learned_kernel.py b/alibi_detect/cd/keops/learned_kernel.py new file mode 100644 index 000000000..acb9a4f43 --- /dev/null +++ b/alibi_detect/cd/keops/learned_kernel.py @@ -0,0 +1,341 @@ +from copy import deepcopy +from functools import partial +from tqdm import tqdm +import numpy as np +from pykeops.torch import LazyTensor +import torch +import torch.nn as nn +from torch.utils.data import DataLoader +from typing import Callable, Dict, List, Optional, Union, Tuple +from alibi_detect.cd.base import BaseLearnedKernelDrift +from alibi_detect.utils.pytorch import get_device, predict_batch +from alibi_detect.utils.pytorch.data import TorchDataset + + +class LearnedKernelDriftKeops(BaseLearnedKernelDrift): + def __init__( + self, + x_ref: Union[np.ndarray, list], + kernel: Union[nn.Module, nn.Sequential], + p_val: float = .05, + x_ref_preprocessed: bool = False, + preprocess_at_init: bool = True, + update_x_ref: Optional[Dict[str, int]] = None, + preprocess_fn: Optional[Callable] = None, + n_permutations: int = 100, + batch_size_permutations: int = 1000000, + var_reg: float = 1e-5, + reg_loss_fn: Callable = (lambda kernel: 0), + train_size: Optional[float] = .75, + retrain_from_scratch: bool = True, + optimizer: torch.optim.Optimizer = torch.optim.Adam, # type: ignore + learning_rate: float = 1e-3, + batch_size: int = 32, + batch_size_predict: int = 1000000, + preprocess_batch_fn: Optional[Callable] = None, + epochs: int = 3, + num_workers: int = 4, + verbose: int = 0, + train_kwargs: Optional[dict] = None, + device: Optional[str] = None, + dataset: Callable = TorchDataset, + dataloader: Callable = DataLoader, + input_shape: Optional[tuple] = None, + data_type: Optional[str] = None + ) -> None: + """ + Maximum Mean Discrepancy (MMD) data drift detector where the kernel is trained to maximise an + estimate of the test power. The kernel is trained on a split of the reference and test instances + and then the MMD is evaluated on held out instances and a permutation test is performed. + + For details see Liu et al (2020): Learning Deep Kernels for Non-Parametric Two-Sample Tests + (https://arxiv.org/abs/2002.09116) + + Parameters + ---------- + x_ref + Data used as reference distribution. + kernel + Trainable PyTorch module that returns a similarity between two instances. + p_val + p-value used for the significance of the test. + x_ref_preprocessed + Whether the given reference data `x_ref` has been preprocessed yet. If `x_ref_preprocessed=True`, only + the test data `x` will be preprocessed at prediction time. If `x_ref_preprocessed=False`, the reference + data will also be preprocessed. + preprocess_at_init + Whether to preprocess the reference data when the detector is instantiated. Otherwise, the reference + data will be preprocessed at prediction time. Only applies if `x_ref_preprocessed=False`. + update_x_ref + Reference data can optionally be updated to the last n instances seen by the detector + or via reservoir sampling with size n. For the former, the parameter equals {'last': n} while + for reservoir sampling {'reservoir_sampling': n} is passed. + preprocess_fn + Function to preprocess the data before applying the kernel. + n_permutations + The number of permutations to use in the permutation test once the MMD has been computed. + batch_size_permutations + KeOps computes the n_permutations of the MMD^2 statistics in chunks of batch_size_permutations. + var_reg + Constant added to the estimated variance of the MMD for stability. + reg_loss_fn + The regularisation term reg_loss_fn(kernel) is added to the loss function being optimized. + train_size + Optional fraction (float between 0 and 1) of the dataset used to train the kernel. + The drift is detected on `1 - train_size`. + retrain_from_scratch + Whether the kernel should be retrained from scratch for each set of test data or whether + it should instead continue training from where it left off on the previous set. + optimizer + Optimizer used during training of the kernel. + learning_rate + Learning rate used by optimizer. + batch_size + Batch size used during training of the kernel. + batch_size_predict + Batch size used for the trained drift detector predictions. + preprocess_batch_fn + Optional batch preprocessing function. For example to convert a list of objects to a batch which can be + processed by the kernel. + epochs + Number of training epochs for the kernel. Corresponds to the smaller of the reference and test sets. + num_workers + Number of workers for the dataloader. + verbose + Verbosity level during the training of the kernel. 0 is silent, 1 a progress bar. + train_kwargs + Optional additional kwargs when training the kernel. + device + Device type used. The default None tries to use the GPU and falls back on CPU if needed. + Can be specified by passing either 'cuda', 'gpu' or 'cpu'. Relevant for 'pytorch' and 'keops' backends. + dataset + Dataset object used during training. + dataloader + Dataloader object used during training. Only relevant for 'pytorch' backend. + input_shape + Shape of input data. + data_type + Optionally specify the data type (tabular, image or time-series). Added to metadata. + """ + super().__init__( + x_ref=x_ref, + p_val=p_val, + x_ref_preprocessed=x_ref_preprocessed, + preprocess_at_init=preprocess_at_init, + update_x_ref=update_x_ref, + preprocess_fn=preprocess_fn, + n_permutations=n_permutations, + train_size=train_size, + retrain_from_scratch=retrain_from_scratch, + input_shape=input_shape, + data_type=data_type + ) + self.meta.update({'backend': 'keops'}) + + # Set device, define model and training kwargs + self.device = get_device(device) + self.original_kernel = kernel + self.kernel = deepcopy(kernel) + + # Check kernel format + self.has_proj = hasattr(self.kernel, 'proj') and isinstance(self.kernel.proj, nn.Module) + self.has_kernel_b = hasattr(self.kernel, 'kernel_b') and isinstance(self.kernel.kernel_b, nn.Module) + + # Define kwargs for dataloader and trainer + self.dataset = dataset + self.dataloader = partial(dataloader, batch_size=batch_size, shuffle=True, + drop_last=True, num_workers=num_workers) + self.train_kwargs = {'optimizer': optimizer, 'epochs': epochs, 'preprocess_fn': preprocess_batch_fn, + 'reg_loss_fn': reg_loss_fn, 'learning_rate': learning_rate, 'verbose': verbose} + if isinstance(train_kwargs, dict): + self.train_kwargs.update(train_kwargs) + + self.j_hat = LearnedKernelDriftKeops.JHat( + self.kernel, var_reg, self.has_proj, self.has_kernel_b).to(self.device) + + # Set prediction and permutation batch sizes + self.batch_size_predict = batch_size_predict + self.batch_size_perms = batch_size_permutations + self.n_batches = 1 + (n_permutations - 1) // batch_size_permutations + + class JHat(nn.Module): + """ + A module that wraps around the kernel. When passed a batch of reference and batch of test + instances it returns an estimate of a correlate of test power. + Equation 4 of https://arxiv.org/abs/2002.09116 + """ + def __init__(self, kernel: nn.Module, var_reg: float, has_proj: bool, has_kernel_b: bool): + super().__init__() + self.kernel = kernel + self.has_proj = has_proj + self.has_kernel_b = has_kernel_b + self.var_reg = var_reg + + def forward(self, x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: + n = len(x) + if self.has_proj and isinstance(self.kernel.proj, nn.Module): + x_proj, y_proj = self.kernel.proj(x), self.kernel.proj(y) + else: + x_proj, y_proj = x, y + x2_proj, x_proj = LazyTensor(x_proj[None, :, :]), LazyTensor(x_proj[:, None, :]) + y2_proj, y_proj = LazyTensor(y_proj[None, :, :]), LazyTensor(y_proj[:, None, :]) + if self.has_kernel_b: + x2, x = LazyTensor(x[None, :, :]), LazyTensor(x[:, None, :]) + y2, y = LazyTensor(y[None, :, :]), LazyTensor(y[:, None, :]) + else: + x, x2, y, y2 = None, None, None, None + + k_xy = self.kernel(x_proj, y2_proj, x, y2) + k_xx = self.kernel(x_proj, x2_proj, x, x2) + k_yy = self.kernel(y_proj, y2_proj, y, y2) + h_mat = k_xx + k_yy - k_xy - k_xy.t() + + h_i = h_mat.sum(1).squeeze(-1) + h = h_i.sum() + mmd2_est = (h - n) / (n * (n - 1)) + var_est = 4 * h_i.square().sum() / (n ** 3) - 4 * h.square() / (n ** 4) + reg_var_est = var_est + self.var_reg + + return mmd2_est/reg_var_est.sqrt() + + def score(self, x: Union[np.ndarray, list]) -> Tuple[float, float, float]: + """ + Compute the p-value resulting from a permutation test using the maximum mean discrepancy + as a distance measure between the reference data and the data to be tested. The kernel + used within the MMD is first trained to maximise an estimate of the resulting test power. + + Parameters + ---------- + x + Batch of instances. + + Returns + ------- + p-value obtained from the permutation test, the MMD^2 between the reference and test set, + and the MMD^2 threshold above which drift is flagged. + """ + x_ref, x_cur = self.preprocess(x) + (x_ref_tr, x_cur_tr), (x_ref_te, x_cur_te) = self.get_splits(x_ref, x_cur) + dl_ref_tr, dl_cur_tr = self.dataloader(self.dataset(x_ref_tr)), self.dataloader(self.dataset(x_cur_tr)) + + self.kernel = deepcopy(self.original_kernel) if self.retrain_from_scratch else self.kernel + self.kernel = self.kernel.to(self.device) + train_args = [self.j_hat, (dl_ref_tr, dl_cur_tr), self.device] + LearnedKernelDriftKeops.trainer(*train_args, **self.train_kwargs) # type: ignore + + m, n = len(x_ref_te), len(x_cur_te) + if isinstance(x_ref_te, np.ndarray) and isinstance(x_cur_te, np.ndarray): + x_all = torch.from_numpy(np.concatenate([x_ref_te, x_cur_te], axis=0)).float() + else: + x_all = x_ref_te + x_cur_te # type: ignore[assignment] + + perms = [torch.randperm(m + n) for _ in range(self.n_permutations)] + mmd2, mmd2_permuted = self._mmd2(x_all, perms, m, n) + if self.device.type == 'cuda': + mmd2, mmd2_permuted = mmd2.cpu(), mmd2_permuted.cpu() + p_val = (mmd2 <= mmd2_permuted).float().mean() + + idx_threshold = int(self.p_val * len(mmd2_permuted)) + distance_threshold = torch.sort(mmd2_permuted, descending=True).values[idx_threshold] + return p_val.numpy().item(), mmd2.numpy().item(), distance_threshold.numpy() + + def _mmd2(self, x_all: Union[list, torch.Tensor], perms: List[torch.Tensor], m: int, n: int) \ + -> Tuple[torch.Tensor, torch.Tensor]: + """ + Batched (across the permutations) MMD^2 computation for the original test statistic and the permutations. + + Parameters + ---------- + x_all + Concatenated reference and test instances. + perms + List with permutation vectors. + m + Number of reference instances. + n + Number of test instances. + + Returns + ------- + MMD^2 statistic for the original and permuted reference and test sets. + """ + preprocess_batch_fn = self.train_kwargs['preprocess_fn'] + if isinstance(preprocess_batch_fn, Callable): # type: ignore[arg-type] + x_all = preprocess_batch_fn(x_all) # type: ignore[operator] + if self.has_proj: + x_all_proj = predict_batch(x_all, self.kernel.proj, device=self.device, batch_size=self.batch_size_predict, + dtype=x_all.dtype if isinstance(x_all, torch.Tensor) else torch.float32) + else: + x_all_proj = x_all + + x, x2, y, y2 = None, None, None, None + k_xx, k_yy, k_xy = [], [], [] + for batch in range(self.n_batches): + i, j = batch * self.batch_size_perms, (batch + 1) * self.batch_size_perms + # Stack a batch of permuted reference and test tensors and their projections + x_proj = torch.cat([x_all_proj[perm[:m]][None, :, :] for perm in perms[i:j]], 0) + y_proj = torch.cat([x_all_proj[perm[m:]][None, :, :] for perm in perms[i:j]], 0) + if self.has_kernel_b: + x = torch.cat([x_all[perm[:m]][None, :, :] for perm in perms[i:j]], 0) + y = torch.cat([x_all[perm[m:]][None, :, :] for perm in perms[i:j]], 0) + if batch == 0: + x_proj = torch.cat([x_all_proj[None, :m, :], x_proj], 0) + y_proj = torch.cat([x_all_proj[None, m:, :], y_proj], 0) + if self.has_kernel_b: + x = torch.cat([x_all[None, :m, :], x], 0) # type: ignore[call-overload] + y = torch.cat([x_all[None, m:, :], y], 0) # type: ignore[call-overload] + x_proj, y_proj = x_proj.to(self.device), y_proj.to(self.device) + if self.has_kernel_b: + x, y = x.to(self.device), y.to(self.device) + + # Batch-wise kernel matrix computation over the permutations + with torch.no_grad(): + x2_proj, x_proj = LazyTensor(x_proj[:, None, :, :]), LazyTensor(x_proj[:, :, None, :]) + y2_proj, y_proj = LazyTensor(y_proj[:, None, :, :]), LazyTensor(y_proj[:, :, None, :]) + if self.has_kernel_b: + x2, x = LazyTensor(x[:, None, :, :]), LazyTensor(x[:, :, None, :]) + y2, y = LazyTensor(y[:, None, :, :]), LazyTensor(y[:, :, None, :]) + k_xy.append(self.kernel(x_proj, y2_proj, x, y2).sum(1).sum(1).squeeze(-1)) + k_xx.append(self.kernel(x_proj, x2_proj, x, x2).sum(1).sum(1).squeeze(-1)) + k_yy.append(self.kernel(y_proj, y2_proj, y, y2).sum(1).sum(1).squeeze(-1)) + + c_xx, c_yy, c_xy = 1 / (m * (m - 1)), 1 / (n * (n - 1)), 2. / (m * n) + # Note that the MMD^2 estimates assume that the diagonal of the kernel matrix consists of 1's + stats = c_xx * (torch.cat(k_xx) - m) + c_yy * (torch.cat(k_yy) - n) - c_xy * torch.cat(k_xy) + return stats[0], stats[1:] + + @staticmethod + def trainer( + j_hat: JHat, + dataloaders: Tuple[DataLoader, DataLoader], + device: torch.device, + optimizer: Callable = torch.optim.Adam, + learning_rate: float = 1e-3, + preprocess_fn: Callable = None, + epochs: int = 20, + reg_loss_fn: Callable = (lambda kernel: 0), + verbose: int = 1, + ) -> None: + """ + Train the kernel to maximise an estimate of test power using minibatch gradient descent. + """ + optimizer = optimizer(j_hat.parameters(), lr=learning_rate) + j_hat.train() + loss_ma = 0. + for epoch in range(epochs): + dl_ref, dl_cur = dataloaders + dl = tqdm(enumerate(zip(dl_ref, dl_cur)), total=min(len(dl_ref), len(dl_cur))) if verbose == 1 else \ + enumerate(zip(dl_ref, dl_cur)) + for step, (x_ref, x_cur) in dl: + if isinstance(preprocess_fn, Callable): # type: ignore + x_ref, x_cur = preprocess_fn(x_ref), preprocess_fn(x_cur) + x_ref, x_cur = x_ref.to(device), x_cur.to(device) + optimizer.zero_grad() # type: ignore + estimate = j_hat(x_ref, x_cur) + loss = -estimate + reg_loss_fn(j_hat.kernel) # ascent + loss.backward() + optimizer.step() # type: ignore + if verbose == 1: + loss_ma = loss_ma + (loss.item() - loss_ma) / (step + 1) + dl.set_description(f'Epoch {epoch + 1}/{epochs}') + dl.set_postfix(dict(loss=loss_ma)) diff --git a/alibi_detect/cd/keops/tests/test_learned_kernel_keops.py b/alibi_detect/cd/keops/tests/test_learned_kernel_keops.py new file mode 100644 index 000000000..646027fe3 --- /dev/null +++ b/alibi_detect/cd/keops/tests/test_learned_kernel_keops.py @@ -0,0 +1,130 @@ +from itertools import product +import numpy as np +import pytest +import torch +import torch.nn as nn +from typing import Callable, Optional, Union +from alibi_detect.utils.frameworks import has_keops +from alibi_detect.utils.pytorch import GaussianRBF as GaussianRBFTorch +from alibi_detect.utils.pytorch import mmd2_from_kernel_matrix +if has_keops: + from alibi_detect.cd.keops.learned_kernel import LearnedKernelDriftKeops + from alibi_detect.utils.keops import GaussianRBF + from pykeops.torch import LazyTensor + +n = 50 # number of instances used for the reference and test data samples in the tests + + +if has_keops: + class MyKernel(nn.Module): + def __init__(self, n_features: int, proj: bool): + super().__init__() + sigma = .1 + self.kernel = GaussianRBF(trainable=True, sigma=torch.Tensor([sigma])) + self.has_proj = proj + if proj: + self.proj = nn.Linear(n_features, 2) + self.kernel_b = GaussianRBF(trainable=True, sigma=torch.Tensor([sigma])) + + def forward(self, x_proj: LazyTensor, y_proj: LazyTensor, x: Optional[LazyTensor] = None, + y: Optional[LazyTensor] = None) -> LazyTensor: + similarity = self.kernel(x_proj, y_proj) + if self.has_proj: + similarity = similarity + self.kernel_b(x, y) + return similarity + + +# test List[Any] inputs to the detector +def identity_fn(x: Union[torch.Tensor, list]) -> torch.Tensor: + if isinstance(x, list): + return torch.from_numpy(np.array(x)) + else: + return x + + +p_val = [.05] +n_features = [4] +preprocess_at_init = [True, False] +update_x_ref = [None, {'reservoir_sampling': 1000}] +preprocess_fn = [None, identity_fn] +n_permutations = [10] +batch_size_permutations = [5, 1000000] +train_size = [.5] +retrain_from_scratch = [True] +batch_size_predict = [1000000] +preprocess_batch = [None, identity_fn] +has_proj = [True, False] +tests_lkdrift = list(product(p_val, n_features, preprocess_at_init, update_x_ref, preprocess_fn, + n_permutations, batch_size_permutations, train_size, retrain_from_scratch, + batch_size_predict, preprocess_batch, has_proj)) +n_tests = len(tests_lkdrift) + + +@pytest.fixture +def lkdrift_params(request): + return tests_lkdrift[request.param] + + +@pytest.mark.skipif(not has_keops, reason='Skipping since pykeops is not installed.') +@pytest.mark.parametrize('lkdrift_params', list(range(n_tests)), indirect=True) +def test_lkdrift(lkdrift_params): + p_val, n_features, preprocess_at_init, update_x_ref, preprocess_fn, \ + n_permutations, batch_size_permutations, train_size, retrain_from_scratch, \ + batch_size_predict, preprocess_batch, has_proj = lkdrift_params + + np.random.seed(0) + torch.manual_seed(0) + + kernel = MyKernel(n_features, has_proj) + x_ref = np.random.randn(*(n, n_features)).astype(np.float32) + x_test1 = np.ones_like(x_ref) + to_list = False + if preprocess_batch is not None and preprocess_fn is None: + to_list = True + x_ref = [_ for _ in x_ref] + update_x_ref = None + + cd = LearnedKernelDriftKeops( + x_ref=x_ref, + kernel=kernel, + p_val=p_val, + preprocess_at_init=preprocess_at_init, + update_x_ref=update_x_ref, + preprocess_fn=preprocess_fn, + n_permutations=n_permutations, + batch_size_permutations=batch_size_permutations, + train_size=train_size, + retrain_from_scratch=retrain_from_scratch, + batch_size_predict=batch_size_predict, + preprocess_batch_fn=preprocess_batch, + batch_size=32, + epochs=1 + ) + + x_test0 = x_ref.copy() + preds_0 = cd.predict(x_test0) + assert cd.n == len(x_test0) + len(x_ref) + assert preds_0['data']['is_drift'] == 0 + + if to_list: + x_test1 = [_ for _ in x_test1] + preds_1 = cd.predict(x_test1) + assert cd.n == len(x_test1) + len(x_test0) + len(x_ref) + assert preds_1['data']['is_drift'] == 1 + assert preds_0['data']['distance'] < preds_1['data']['distance'] + + # ensure the keops MMD^2 estimate matches the pytorch implementation for the same kernel + if not isinstance(x_ref, list) and update_x_ref is None and not has_proj: + if isinstance(preprocess_fn, Callable): + x_ref, x_test1 = cd.preprocess(x_test1) + n_ref, n_test = x_ref.shape[0], x_test1.shape[0] + x_all = torch.from_numpy(np.concatenate([x_ref, x_test1], axis=0)).float() + perms = [torch.randperm(n_ref + n_test) for _ in range(n_permutations)] + mmd2 = cd._mmd2(x_all, perms, n_ref, n_test)[0] + + if isinstance(preprocess_batch, Callable): + x_all = preprocess_batch(x_all) + kernel = GaussianRBFTorch(sigma=cd.kernel.kernel.sigma) + kernel_mat = kernel(x_all, x_all) + mmd2_torch = mmd2_from_kernel_matrix(kernel_mat, n_test) + np.testing.assert_almost_equal(mmd2, mmd2_torch, decimal=6) diff --git a/alibi_detect/cd/learned_kernel.py b/alibi_detect/cd/learned_kernel.py index 308b7a2fa..81e3110c4 100644 --- a/alibi_detect/cd/learned_kernel.py +++ b/alibi_detect/cd/learned_kernel.py @@ -1,6 +1,6 @@ import numpy as np from typing import Callable, Dict, Optional, Union -from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, BackendValidator +from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, has_keops, BackendValidator from alibi_detect.utils.warnings import deprecated_alias from alibi_detect.base import DriftConfigMixin @@ -13,6 +13,9 @@ from alibi_detect.cd.tensorflow.learned_kernel import LearnedKernelDriftTF from alibi_detect.utils.tensorflow.data import TFDataset +if has_keops: + from alibi_detect.cd.keops.learned_kernel import LearnedKernelDriftKeops + class LearnedKernelDrift(DriftConfigMixin): @deprecated_alias(preprocess_x_ref='preprocess_at_init') @@ -27,6 +30,7 @@ def __init__( update_x_ref: Optional[Dict[str, int]] = None, preprocess_fn: Optional[Callable] = None, n_permutations: int = 100, + batch_size_permutations: int = 1000000, var_reg: float = 1e-5, reg_loss_fn: Callable = (lambda kernel: 0), train_size: Optional[float] = .75, @@ -34,6 +38,7 @@ def __init__( optimizer: Optional[Callable] = None, learning_rate: float = 1e-3, batch_size: int = 32, + batch_size_predict: int = 1000000, preprocess_batch_fn: Optional[Callable] = None, epochs: int = 3, verbose: int = 0, @@ -52,7 +57,6 @@ def __init__( For details see Liu et al (2020): Learning Deep Kernels for Non-Parametric Two-Sample Tests (https://arxiv.org/abs/2002.09116) - Parameters ---------- x_ref @@ -78,6 +82,9 @@ def __init__( Function to preprocess the data before applying the kernel. n_permutations The number of permutations to use in the permutation test once the MMD has been computed. + batch_size_permutations + KeOps computes the n_permutations of the MMD^2 statistics in chunks of batch_size_permutations. + Only relevant for 'keops' backend. var_reg Constant added to the estimated variance of the MMD for stability. reg_loss_fn @@ -94,6 +101,8 @@ def __init__( Learning rate used by optimizer. batch_size Batch size used during training of the kernel. + batch_size_predict + Batch size used for the trained drift detector predictions. Only relevant for 'keops' backend. preprocess_batch_fn Optional batch preprocessing function. For example to convert a list of objects to a batch which can be processed by the kernel. @@ -105,11 +114,11 @@ def __init__( Optional additional kwargs when training the kernel. device Device type used. The default None tries to use the GPU and falls back on CPU if needed. - Can be specified by passing either 'cuda', 'gpu' or 'cpu'. Only relevant for 'pytorch' backend. + Can be specified by passing either 'cuda', 'gpu' or 'cpu'. Relevant for 'pytorch' and 'keops' backends. dataset Dataset object used during training. dataloader - Dataloader object used during training. Only relevant for 'pytorch' backend. + Dataloader object used during training. Relevant for 'pytorch' and 'keops' backends. input_shape Shape of input data. data_type @@ -123,7 +132,8 @@ def __init__( backend = backend.lower() BackendValidator( backend_options={'tensorflow': ['tensorflow'], - 'pytorch': ['pytorch']}, + 'pytorch': ['pytorch'], + 'keops': ['keops']}, construct_name=self.__class__.__name__ ).verify_backend(backend) @@ -134,18 +144,25 @@ def __init__( pop_kwargs += ['optimizer'] [kwargs.pop(k, None) for k in pop_kwargs] - if backend == 'tensorflow' and has_tensorflow: - pop_kwargs = ['device', 'dataloader'] + if backend == 'tensorflow': + pop_kwargs = ['device', 'dataloader', 'batch_size_permutations', 'batch_size_predict'] [kwargs.pop(k, None) for k in pop_kwargs] if dataset is None: kwargs.update({'dataset': TFDataset}) - self._detector = LearnedKernelDriftTF(*args, **kwargs) # type: ignore + detector = LearnedKernelDriftTF else: if dataset is None: kwargs.update({'dataset': TorchDataset}) if dataloader is None: kwargs.update({'dataloader': DataLoader}) - self._detector = LearnedKernelDriftTorch(*args, **kwargs) # type: ignore + if backend == 'pytorch': + pop_kwargs = ['batch_size_permutations', 'batch_size_predict'] + [kwargs.pop(k, None) for k in pop_kwargs] + detector = LearnedKernelDriftTorch + else: + detector = LearnedKernelDriftKeops + + self._detector = detector(*args, **kwargs) # type: ignore self.meta = self._detector.meta def predict(self, x: Union[np.ndarray, list], return_p_val: bool = True, diff --git a/alibi_detect/cd/tests/test_learned_kernel.py b/alibi_detect/cd/tests/test_learned_kernel.py index 86cb960d2..e14565bf6 100644 --- a/alibi_detect/cd/tests/test_learned_kernel.py +++ b/alibi_detect/cd/tests/test_learned_kernel.py @@ -7,6 +7,10 @@ from alibi_detect.cd import LearnedKernelDrift from alibi_detect.cd.pytorch.learned_kernel import LearnedKernelDriftTorch from alibi_detect.cd.tensorflow.learned_kernel import LearnedKernelDriftTF +from alibi_detect.utils.frameworks import has_keops +if has_keops: + from alibi_detect.cd.keops.learned_kernel import LearnedKernelDriftKeops + from pykeops.torch import LazyTensor n, n_features = 100, 5 @@ -37,7 +41,16 @@ def forward(self, x: torch.Tensor, y: torch.Tensor) -> torch.Tensor: return torch.einsum('ji,ki->jk', self.dense(x), self.dense(y)) -tests_lkdrift = ['tensorflow', 'pytorch', 'PyToRcH', 'mxnet'] +if has_keops: + class MyKernelKeops(nn.Module): + def __init__(self): + super().__init__() + + def forward(self, x: LazyTensor, y: LazyTensor) -> LazyTensor: + return (- ((x - y) ** 2).sum(-1)).exp() + + +tests_lkdrift = ['tensorflow', 'pytorch', 'keops', 'PyToRcH', 'mxnet'] n_tests = len(tests_lkdrift) @@ -53,6 +66,8 @@ def test_lkdrift(lkdrift_params): kernel = MyKernelTorch(n_features) elif backend.lower() == 'tensorflow': kernel = MyKernelTF(n_features) + elif has_keops and backend.lower() == 'keops': + kernel = MyKernelKeops() else: kernel = None x_ref = np.random.randn(*(n, n_features)) @@ -61,10 +76,15 @@ def test_lkdrift(lkdrift_params): cd = LearnedKernelDrift(x_ref=x_ref, kernel=kernel, backend=backend) except NotImplementedError: cd = None + except ImportError: + assert not has_keops + cd = None if backend.lower() == 'pytorch': assert isinstance(cd._detector, LearnedKernelDriftTorch) elif backend.lower() == 'tensorflow': assert isinstance(cd._detector, LearnedKernelDriftTF) + elif has_keops and backend.lower() == 'keops': + assert isinstance(cd._detector, LearnedKernelDriftKeops) else: assert cd is None diff --git a/alibi_detect/saving/schemas.py b/alibi_detect/saving/schemas.py index 6649fffb2..196c8e121 100644 --- a/alibi_detect/saving/schemas.py +++ b/alibi_detect/saving/schemas.py @@ -842,6 +842,7 @@ class LearnedKernelDriftConfig(DriftDetectorConfig): preprocess_at_init: bool = True update_x_ref: Optional[Dict[str, int]] = None n_permutations: int = 100 + batch_size_permutations: int = 1000000 var_reg: float = 1e-5 reg_loss_fn: Optional[str] = None train_size: Optional[float] = .75 @@ -849,6 +850,7 @@ class LearnedKernelDriftConfig(DriftDetectorConfig): optimizer: Optional[Union[str, OptimizerConfig]] = None learning_rate: float = 1e-3 batch_size: int = 32 + batch_size_predict: int = 1000000 preprocess_batch_fn: Optional[str] = None epochs: int = 3 verbose: int = 0 @@ -872,6 +874,7 @@ class LearnedKernelDriftConfigResolved(DriftDetectorConfigResolved): preprocess_at_init: bool = True update_x_ref: Optional[Dict[str, int]] = None n_permutations: int = 100 + batch_size_permutations: int = 1000000 var_reg: float = 1e-5 reg_loss_fn: Optional[Callable] = None train_size: Optional[float] = .75 @@ -879,6 +882,7 @@ class LearnedKernelDriftConfigResolved(DriftDetectorConfigResolved): optimizer: Optional['tf.keras.optimizers.Optimizer'] = None learning_rate: float = 1e-3 batch_size: int = 32 + batch_size_predict: int = 1000000 preprocess_batch_fn: Optional[Callable] = None epochs: int = 3 verbose: int = 0 diff --git a/alibi_detect/tests/test_dep_management.py b/alibi_detect/tests/test_dep_management.py index 4333d93c9..bb1bbc494 100644 --- a/alibi_detect/tests/test_dep_management.py +++ b/alibi_detect/tests/test_dep_management.py @@ -281,6 +281,7 @@ def test_keops_utils_dependencies(opt_dep): dependency_map = defaultdict(lambda: ['default']) for dependency, relations in [ ("GaussianRBF", ['keops']), + ("DeepKernel", ['keops']), ]: dependency_map[dependency] = relations from alibi_detect.utils import keops as keops_utils diff --git a/alibi_detect/utils/keops/__init__.py b/alibi_detect/utils/keops/__init__.py index 9da9f5073..36dc22971 100644 --- a/alibi_detect/utils/keops/__init__.py +++ b/alibi_detect/utils/keops/__init__.py @@ -1,8 +1,12 @@ from alibi_detect.utils.missing_optional_dependency import import_optional -GaussianRBF = import_optional('alibi_detect.utils.keops.kernels', names=['GaussianRBF']) +GaussianRBF, DeepKernel = import_optional( + 'alibi_detect.utils.keops.kernels', + names=['GaussianRBF', 'DeepKernel'] +) __all__ = [ - "GaussianRBF" + "GaussianRBF", + "DeepKernel" ] diff --git a/alibi_detect/utils/keops/kernels.py b/alibi_detect/utils/keops/kernels.py index 380ddadb9..5e1f6bb53 100644 --- a/alibi_detect/utils/keops/kernels.py +++ b/alibi_detect/utils/keops/kernels.py @@ -1,10 +1,10 @@ from pykeops.torch import LazyTensor import torch import torch.nn as nn -from typing import Callable, Optional +from typing import Callable, Optional, Union -def sigma_mean(x: LazyTensor, y: LazyTensor, dist: LazyTensor, n_min: int = None) -> torch.Tensor: +def sigma_mean(x: LazyTensor, y: LazyTensor, dist: LazyTensor, n_min: int = 100) -> torch.Tensor: """ Set bandwidth to the mean distance between instances x and y. @@ -25,8 +25,8 @@ def sigma_mean(x: LazyTensor, y: LazyTensor, dist: LazyTensor, n_min: int = None consists of all zeros. We do this by computing the k-min distances and k-argmin indices over the columns of the distance matrix. We then check if the distances on the diagonal of the distance matrix are all zero or not. If they are all zero, then we do not use these distances (zeros) when computing - the mean pairwise distance as bandwidth. The default `None` sets k to Nx (=Ny). If Nx becomes very large, - it is advised to set `n_min` to a lower value. + the mean pairwise distance as bandwidth. If Nx becomes very large, it is advised to set `n_min` + to a low enough value to avoid OOM issues. By default we set it to 100 instances. Returns ------- @@ -41,7 +41,7 @@ def sigma_mean(x: LazyTensor, y: LazyTensor, dist: LazyTensor, n_min: int = None axis = 2 n_mean = nx * ny if nx == ny: - n_min = n_min if isinstance(n_min, int) else nx + n_min = min(n_min, nx) if isinstance(n_min, int) else nx d_min, id_min = dist.Kmin_argKmin(n_min, axis=axis) if batched: d_min, id_min = d_min[0], id_min[0] # first instance in permutation test contains the original data @@ -114,3 +114,65 @@ def forward(self, x: LazyTensor, y: LazyTensor, infer_sigma: bool = False) -> La if len(dist.shape) < len(gamma.shape): kernel_mat = kernel_mat.sum(-1) / len(self.sigma) return kernel_mat + + +class DeepKernel(nn.Module): + def __init__( + self, + proj: nn.Module, + kernel_a: nn.Module = GaussianRBF(trainable=True), + kernel_b: Optional[nn.Module] = GaussianRBF(trainable=True), + eps: Union[float, str] = 'trainable' + ) -> None: + """ + Computes similarities as k(x,y) = (1-eps)*k_a(proj(x), proj(y)) + eps*k_b(x,y). + A forward pass takes an already projected batch of instances x_proj and y_proj and optionally + (if k_b is present) a batch of instances x and y and returns the kernel matrix. + x_proj can be of shape [Nx, 1, features_proj] or [batch_size, Nx, 1, features_proj]. + y_proj can be of shape [1, Ny, features_proj] or [batch_size, 1, Ny, features_proj]. + x can be of shape [Nx, 1, features] or [batch_size, Nx, 1, features]. + y can be of shape [1, Ny, features] or [batch_size, 1, Ny, features]. + The returned kernel matrix can be of shape [Nx, Ny] or [batch_size, Nx, Ny]. + x, y and the returned kernel matrix are all lazy tensors. + + Parameters + ---------- + proj + The projection to be applied to the inputs before applying kernel_a + kernel_a + The kernel to apply to the projected inputs. Defaults to a Gaussian RBF with trainable bandwidth. + kernel_b + The kernel to apply to the raw inputs. Defaults to a Gaussian RBF with trainable bandwidth. + Set to None in order to use only the deep component (i.e. eps=0). + eps + The proportion (in [0,1]) of weight to assign to the kernel applied to raw inputs. This can be + either specified or set to 'trainable'. Only relavent if kernel_b is not None. + """ + super().__init__() + + self.kernel_a = kernel_a + self.kernel_b = kernel_b + self.proj = proj + if kernel_b is not None: + self._init_eps(eps) + + def _init_eps(self, eps: Union[float, str]) -> None: + if isinstance(eps, float): + if not 0 < eps < 1: + raise ValueError("eps should be in (0,1)") + self.logit_eps = nn.Parameter(torch.tensor(eps).logit(), requires_grad=False) + elif eps == 'trainable': + self.logit_eps = nn.Parameter(torch.tensor(0.)) + else: + raise NotImplementedError("eps should be 'trainable' or a float in (0,1)") + + @property + def eps(self) -> torch.Tensor: + return self.logit_eps.sigmoid() if self.kernel_b is not None else torch.tensor(0.) + + def forward(self, x_proj: LazyTensor, y_proj: LazyTensor, x: Optional[LazyTensor] = None, + y: Optional[LazyTensor] = None) -> LazyTensor: + similarity = self.kernel_a(x_proj, y_proj) + if self.kernel_b is not None: + similarity = (1-self.eps)*similarity + self.eps*self.kernel_b(x, y) + return similarity diff --git a/alibi_detect/utils/keops/tests/test_kernels_keops.py b/alibi_detect/utils/keops/tests/test_kernels_keops.py index 0c1489410..b25554818 100644 --- a/alibi_detect/utils/keops/tests/test_kernels_keops.py +++ b/alibi_detect/utils/keops/tests/test_kernels_keops.py @@ -3,9 +3,10 @@ from alibi_detect.utils.frameworks import has_keops import pytest import torch +import torch.nn as nn if has_keops: from pykeops.torch import LazyTensor - from alibi_detect.utils.keops import GaussianRBF + from alibi_detect.utils.keops import DeepKernel, GaussianRBF sigma = [None, np.array([1.]), np.array([1., 2.])] n_features = [5, 10] @@ -26,8 +27,6 @@ def gaussian_kernel_params(request): def test_gaussian_kernel(gaussian_kernel_params): sigma, n_features, n_instances, batch_size, trainable = gaussian_kernel_params - print(sigma, n_features, n_instances, batch_size, trainable) - xshape, yshape = (n_instances[0], n_features), (n_instances[1], n_features) if batch_size: xshape = (batch_size, ) + xshape @@ -64,3 +63,59 @@ def test_gaussian_kernel(gaussian_kernel_params): k_xx_argmax, k_xx_min, k_xy_min = k_xx_argmax[0], k_xx_min[0], k_xy_min[0] assert (torch.arange(n_instances[0]) == k_xx_argmax.cpu().view(-1)).all() assert (k_xx_min >= 0.).all() and (k_xy_min >= 0.).all() + + +if has_keops: + class MyKernel(nn.Module): + def __init__(self): + super().__init__() + + def forward(self, x: LazyTensor, y: LazyTensor) -> LazyTensor: + return (- ((x - y) ** 2).sum(-1)).exp() + + +n_features = [5] +n_instances = [(100, 100), (100, 75)] +kernel_a = ['GaussianRBF', 'MyKernel'] +kernel_b = ['GaussianRBF', 'MyKernel', None] +eps = [0.5, 'trainable'] +tests_dk = list(product(n_features, n_instances, kernel_a, kernel_b, eps)) +n_tests_dk = len(tests_dk) + + +@pytest.fixture +def deep_kernel_params(request): + return tests_dk[request.param] + + +@pytest.mark.skipif(not has_keops, reason='Skipping since pykeops is not installed.') +@pytest.mark.parametrize('deep_kernel_params', list(range(n_tests_dk)), indirect=True) +def test_deep_kernel(deep_kernel_params): + n_features, n_instances, kernel_a, kernel_b, eps = deep_kernel_params + + proj = nn.Linear(n_features, n_features) + kernel_a = MyKernel() if kernel_a == 'MyKernel' else GaussianRBF(trainable=True) + if kernel_b == 'MyKernel': + kernel_b = MyKernel() + elif kernel_b == 'GaussianRBF': + kernel_b = GaussianRBF(trainable=True) + kernel = DeepKernel(proj, kernel_a=kernel_a, kernel_b=kernel_b, eps=eps) + + xshape, yshape = (n_instances[0], n_features), (n_instances[1], n_features) + x = torch.as_tensor(np.random.random(xshape).astype('float32')) + y = torch.as_tensor(np.random.random(yshape).astype('float32')) + x_proj, y_proj = kernel.proj(x), kernel.proj(y) + x2_proj, x_proj = LazyTensor(x_proj[None, :, :]), LazyTensor(x_proj[:, None, :]) + y2_proj, y_proj = LazyTensor(y_proj[None, :, :]), LazyTensor(y_proj[:, None, :]) + if kernel_b: + x2, x = LazyTensor(x[None, :, :]), LazyTensor(x[:, None, :]) + y2, y = LazyTensor(y[None, :, :]), LazyTensor(y[:, None, :]) + else: + x, x2, y, y2 = None, None, None, None + + k_xy = kernel(x_proj, y2_proj, x, y2) + k_yx = kernel(y_proj, x2_proj, y, x2) + k_xx = kernel(x_proj, x2_proj, x, x2) + assert k_xy.shape == n_instances and k_xx.shape == (xshape[0], xshape[0]) + assert (k_xx.Kmin_argKmin(1, axis=1)[0] > 0.).all() + assert (torch.abs(k_xy.sum(1).sum(1) - k_yx.t().sum(1).sum(1)) < 1e-5).all() diff --git a/doc/source/cd/methods/learnedkerneldrift.ipynb b/doc/source/cd/methods/learnedkerneldrift.ipynb index 0f4528c6d..3ae9790d8 100644 --- a/doc/source/cd/methods/learnedkerneldrift.ipynb +++ b/doc/source/cd/methods/learnedkerneldrift.ipynb @@ -34,12 +34,11 @@ "\n", "* `x_ref`: Data used as reference distribution.\n", "\n", - "* `kernel`: A differentiable **TensorFlow** or **PyTorch** module that takes two instances as input and returns a scalar notion of similarity as output.\n", - "\n", + "* `kernel`: A differentiable **TensorFlow** or **PyTorch** module that takes two sets of instances as inputs and returns a kernel similarity matrix as output.\n", "\n", "Keyword arguments:\n", "\n", - "* `backend`: Specify the backend (*tensorflow* or *pytorch*). This depends on the framework of the `kernel`. Defaults to *tensorflow*.\n", + "* `backend`: **TensorFlow**, **PyTorch** and [**KeOps**](https://github.com/getkeops/keops) implementations of the learned kernel detector are available. The backend can be specified as *tensorflow*, *pytorch* or *keops*. Defaults to *tensorflow*.\n", "\n", "* `p_val`: p-value threshold used for the significance of the test.\n", "\n", @@ -55,11 +54,11 @@ "\n", "* `var_reg`: Constant added to the estimated variance of the MMD for stability.\n", "\n", - "* `reg_loss_fn`: The regularisation term reg_loss_fn(kernel) is added to the loss function being optimized.\n", + "* `reg_loss_fn`: The regularisation term *reg_loss_fn(kernel)* is added to the loss function being optimized.\n", "\n", "* `train_size`: Optional fraction (float between 0 and 1) of the dataset used to train the classifier. The drift is detected on *1 - train_size*.\n", "\n", - "* `retrain_from_scratch`: Whether the kernel should be retrained from scratch for each set of test data or whether it should instead continue training from where it left off on the previous set.\n", + "* `retrain_from_scratch`: Whether the kernel should be retrained from scratch for each set of test data or whether it should instead continue training from where it left off on the previous set. Defaults to *True*.\n", "\n", "* `optimizer`: Optimizer used during training of the kernel. From `torch.optim` for PyTorch and `tf.keras.optimizers` for TensorFlow.\n", "\n", @@ -75,7 +74,7 @@ "\n", "* `train_kwargs`: Optional additional kwargs for the built-in TensorFlow (`from alibi_detect.models.tensorflow import trainer`) or PyTorch (`from alibi_detect.models.pytorch import trainer`) trainer functions.\n", "\n", - "* `dataset`: Dataset object used during training of the kernel. Defaults to `alibi_detect.utils.pytorch.TorchDataset` (an instance of `torch.utils.data.Dataset`) for the PyTorch backend and `alibi_detect.utils.tensorflow.TFDataset` (an instance of `tf.keras.utils.Sequence`) for the TensorFlow backend. For PyTorch, the dataset should only take the windows x_ref and x_test as input, so when e.g. *TorchDataset* is passed to the detector at initialisation, during training *TorchDataset(x_ref, x_test)* is used. For TensorFlow, the dataset is an instance of `tf.keras.utils.Sequence`, so when e.g. *TFDataset* is passed to the detector at initialisation, during training *TFDataset(x_ref, x_test, batch_size=batch_size, shuffle=True)* is used. x_ref and x_test can be of type np.ndarray or List[Any].\n", + "* `dataset`: Dataset object used during training of the kernel. Defaults to `alibi_detect.utils.pytorch.TorchDataset` (an instance of `torch.utils.data.Dataset`) for the PyTorch and KeOps backends and `alibi_detect.utils.tensorflow.TFDataset` (an instance of `tf.keras.utils.Sequence`) for the TensorFlow backend. For PyTorch or KeOps, the dataset should only take the windows x_ref and x_test as input, so when e.g. *TorchDataset* is passed to the detector at initialisation, during training *TorchDataset(x_ref, x_test)* is used. For TensorFlow, the dataset is an instance of `tf.keras.utils.Sequence`, so when e.g. *TFDataset* is passed to the detector at initialisation, during training *TFDataset(x_ref, x_test, batch_size=batch_size, shuffle=True)* is used. x_ref and x_test can be of type np.ndarray or List[Any].\n", "\n", "* `input_shape`: Shape of input data.\n", "\n", @@ -88,9 +87,15 @@ "\n", "* `dataloader`: Dataloader object used during training of the kernel. Defaults to `torch.utils.data.DataLoader`. The dataloader is not initialized yet, this is done during init off the detector using the `batch_size`. Custom dataloaders can be passed as well, e.g. for graph data we can use `torch_geometric.data.DataLoader`.\n", "\n", + "Additional KeOps keyword arguments:\n", + "\n", + "* `batch_size_permutations`: KeOps computes the `n_permutations` of the MMD^2 statistics in chunks of `batch_size_permutations`. Defaults to 1,000,000.\n", + "\n", + "* `batch_size_predict`: Batch size used for the trained drift detector predictions. Defaults to 1,000,000.\n", + "\n", "### Defining the kernel\n", "\n", - "Any differentiable *Pytorch* or *TensorFlow* module that takes as input two instances and outputs a scalar (representing similarity) can be used as the kernel for this drift detector. However, in order to ensure that MMD=0 implies no-drift the kernel should satify a *characteristic* property. This can be guarenteed by defining a kernel as $$k(x,y)=(1-\\epsilon)*k_a(\\Phi(x), \\Phi(y)) + \\epsilon*k_b(x,y),$$ where $\\Phi$ is a learnable projection, $k_a$ and $k_b$ are simple characteristic kernels (such as a [Gaussian RBF](https://en.wikipedia.org/wiki/Radial_basis_function_kernel)), and $\\epsilon>0$ is a small constant. By letting $\\Phi$ be very flexible we can learn powerful kernels in this manner.\n", + "Any differentiable *Pytorch* or *TensorFlow* module that takes as input two instances and outputs a scalar (representing similarity) can be used as the kernel for this drift detector. However, in order to ensure that MMD=0 implies no-drift the kernel should satify a *characteristic* property. This can be guaranteed by defining a kernel as $$k(x,y)=(1-\\epsilon)*k_a(\\Phi(x), \\Phi(y)) + \\epsilon*k_b(x,y),$$ where $\\Phi$ is a learnable projection, $k_a$ and $k_b$ are simple characteristic kernels (such as a [Gaussian RBF](https://en.wikipedia.org/wiki/Radial_basis_function_kernel)), and $\\epsilon>0$ is a small constant. By letting $\\Phi$ be very flexible we can learn powerful kernels in this manner.\n", "\n", "This is easily implemented using the `DeepKernel` class provided in `alibi_detect`. We demonstrate below how we might define a convolutional kernel for images using *Pytorch*. By default `GaussianRBF` kernels are used for $k_a$ and $k_b$ and here we specify $\\epsilon=0.01$, but we could alternatively set `eps='trainable'`.\n", "\n", @@ -113,6 +118,22 @@ "kernel = DeepKernel(proj, eps=0.01)\n", "```\n", "\n", + "It is important to note that, if `retrain_from_scratch=True` and we have not initialised the kernel bandwidth `sigma` for the default `GaussianRBF` kernel $k_a$ and optionally also for $k_b$, we will initialise `sigma` using a median (*PyTorch* and *TensorFlow*) or mean (*KeOps*) bandwidth heuristic for every detector prediction. For KeOps detectors specifically, this could form a computational bottleneck and should be avoided by already specifying a bandwidth in advance. To do this, we can leverage the library's built-in heuristics:\n", + "\n", + "```python\n", + "from alibi_detect.utils.pytorch.kernels import sigma_median, GaussianRBF\n", + "\n", + "# example usage\n", + "x, y = torch.randn(*shape), torch.randn(*shape)\n", + "dist = ((x[:, None, :] - y[None, :, :]) ** 2).sum(-1) # distance used for the GaussianRBF kernel\n", + "sigma = sigma_median(x, y, dist)\n", + "kernel_b = GaussianRBF(sigma=sigma, trainable=True)\n", + "\n", + "# equivalent TensorFlow and KeOps functions\n", + "from alibi_detect.utils.tensorflow.kernels import sigma_median\n", + "from alibi_detect.utils.keops.kernels import sigma_mean\n", + "```\n", + "\n", "### Instantiating the detector\n", "\n", "Instantiating the detector is then as simple as passing the reference data and the kernel as follows:\n", @@ -123,8 +144,16 @@ "cd = LearnedKernelDrift(x_ref, kernel, backend='pytorch', p_val=.05, epochs=10, batch_size=32)\n", "```\n", "\n", + "We could have alternatively defined the kernel and instantiated the detector using *KeOps*:\n", + "\n", + "```python\n", + "from alibi_detect.utils.keops import DeepKernel\n", + "\n", + "kernel = DeepKernel(proj, eps=0.01)\n", + "cd = LearnedKernelDrift(x_ref, kernel, backend='keops', p_val=.05, epochs=10, batch_size=32)\n", + "```\n", "\n", - "We could have alternatively defined the kernel and instantiated the detector using *TensorFlow*:\n", + "Or by using *TensorFlow* as the backend:\n", "\n", "```python\n", "import tensorflow as tf\n", @@ -190,7 +219,11 @@ "\n", "### Image\n", "\n", - "[Drift detection on CIFAR10](../../examples/cd_clf_cifar10.ipynb)" + "[Drift detection on CIFAR10](../../examples/cd_clf_cifar10.ipynb)\n", + "\n", + "### Tabular\n", + "\n", + "[Scaling up drift detection with KeOps](../../examples/cd_mmd_keops.ipynb)" ] } ], diff --git a/doc/source/examples/cd_mmd_keops.ipynb b/doc/source/examples/cd_mmd_keops.ipynb index dac1556e8..3b5ea1f57 100644 --- a/doc/source/examples/cd_mmd_keops.ipynb +++ b/doc/source/examples/cd_mmd_keops.ipynb @@ -2,20 +2,15 @@ "cells": [ { "cell_type": "markdown", - "id": "27a4394b", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ "# Scaling up drift detection with KeOps\n", "\n", "## Introduction\n", "\n", - "A number of convenient and powerful kernel-based drift detectors such as the MMD detector ([Gretton et al., 2012](https://jmlr.csail.mit.edu/papers/v13/gretton12a.html)) do not scale favourably with increasing dataset size $n$, leading to quadratic complexity $\\mathcal{O}(n^2)$ for naive implementations. As a result, we can quickly run into memory issues by having to store the $[N_\\text{ref} + N_\\text{test}, N_\\text{ref} + N_\\text{test}]$ kernel matrix (on the GPU if applicable) used for an efficient implementation of the permutation test. Note that $N_\\text{ref}$ is the reference data size and $N_\\text{test}$ the test data size.\n", + "A number of convenient and powerful kernel-based drift detectors such as the [MMD detector](https://docs.seldon.io/projects/alibi-detect/en/stable/cd/methods/mmddrift.html) ([Gretton et al., 2012](https://jmlr.csail.mit.edu/papers/v13/gretton12a.html)) or the [learned kernel MMD detector](https://docs.seldon.io/projects/alibi-detect/en/stable/cd/methods/learnedkerneldrift.html) ([Liu et al., 2020](https://arxiv.org/abs/2002.09116)) do not scale favourably with increasing dataset size $n$, leading to quadratic complexity $\\mathcal{O}(n^2)$ for naive implementations. As a result, we can quickly run into memory issues by having to store the $[N_\\text{ref} + N_\\text{test}, N_\\text{ref} + N_\\text{test}]$ kernel matrix (on the GPU if applicable) used for an efficient implementation of the permutation test. Note that $N_\\text{ref}$ is the reference data size and $N_\\text{test}$ the test data size.\n", "\n", - "We can however drastically speed up and scale up kernel-based drift detectors to large dataset sizes by working with symbolic kernel matrices instead and leverage the [KeOps](https://www.kernel-operations.io/keops/index.html) library to do so. For the user of $\\texttt{Alibi Detect}$ the only thing that changes is the specification of the detector's backend:\n", + "We can however drastically speed up and scale up kernel-based drift detectors to large dataset sizes by working with symbolic kernel matrices instead and leverage the [KeOps](https://www.kernel-operations.io/keops/index.html) library to do so. For the user of $\\texttt{Alibi Detect}$ the only thing that changes is the specification of the detector's backend, e.g. for the MMD detector:\n", "\n", "\n", "```python\n", @@ -25,11 +20,11 @@ "detector_keops = MMDDrift(x_ref, backend='keops')\n", "```\n", "\n", - "In this notebook we will run a few simple benchmarks to illustrate the speed and memory improvements from using KeOps over vanilla PyTorch on the GPU (1x RTX 2080 Ti).\n", + "In this notebook we will run a few simple benchmarks to illustrate the speed and memory improvements from using KeOps over vanilla PyTorch on the GPU (1x RTX 2080 Ti) for both the standard MMD and learned kernel MMD detectors.\n", "\n", "## Data\n", "\n", - "We randomly sample points from the standard normal distribution and run the MMD detectors with PyTorch and KeOps backends for the following settings:\n", + "We randomly sample points from the standard normal distribution and run the detectors with PyTorch and KeOps backends for the following settings:\n", "\n", "- $N_\\text{ref}, N_\\text{test} = [2, 5, 10, 20, 50, 100]$ (batch sizes in '000s)\n", "- $D = [2, 10, 50]$\n", @@ -38,31 +33,21 @@ "\n", "## Requirements\n", "\n", - "The notebook requires [PyTorch](https://pytorch.org/) and KeOps to be installed. These are optional dependencies for $\\texttt{Alibi Detect}$ and can be installed using:" + "The notebook requires [PyTorch](https://pytorch.org/) and KeOps to be installed. Once PyTorch is installed, KeOps can be installed via pip:" ] }, { "cell_type": "code", "execution_count": null, - "id": "a0bf1719", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "metadata": {}, "outputs": [], "source": [ - "!pip install alibi-detect[keops]" + "!pip install pykeops" ] }, { "cell_type": "markdown", - "id": "7ff93d59", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ "Before we start let’s fix the random seeds for reproducibility:" ] @@ -70,12 +55,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "2ba95f29", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "metadata": {}, "outputs": [], "source": [ "import numpy as np\n", @@ -91,17 +71,12 @@ }, { "cell_type": "markdown", - "id": "1910895a", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ "\n", "## Vanilla PyTorch vs. KeOps comparison\n", "\n", - "### Experiments\n", + "### Utility functions\n", "\n", "First we define some utility functions to run the experiments:" ] @@ -109,19 +84,30 @@ { "cell_type": "code", "execution_count": 2, - "id": "a1c65254", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "metadata": {}, "outputs": [], "source": [ - "from alibi_detect.cd import MMDDrift\n", + "from alibi_detect.cd import MMDDrift, LearnedKernelDrift\n", + "from alibi_detect.utils.keops.kernels import DeepKernel as DeepKernelKeops\n", + "from alibi_detect.utils.keops.kernels import GaussianRBF as GaussianRBFKeops\n", + "from alibi_detect.utils.pytorch.kernels import DeepKernel as DeepKernelTorch\n", + "from alibi_detect.utils.pytorch.kernels import GaussianRBF as GaussianRBFTorch\n", "import matplotlib.pyplot as plt\n", "from scipy.stats import kstest\n", "from timeit import default_timer as timer\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "\n", "\n", + "class Projection(nn.Module):\n", + " def __init__(self, d_in: int, d_out: int = 2):\n", + " super().__init__()\n", + " self.lin1 = nn.Linear(d_in, d_out)\n", + " self.lin2 = nn.Linear(d_out, d_out)\n", + " \n", + " def forward(self, x):\n", + " return self.lin2(F.relu(self.lin1(x)))\n", + " \n", "\n", "def eval_detector(p_vals: np.ndarray, threshold: float, is_drift: bool, t_mean: float, t_std: float) -> dict:\n", " \"\"\" In case of drifted data (ground truth) it returns the detector's power.\n", @@ -140,7 +126,8 @@ " return results\n", "\n", "\n", - "def experiment(backend: str, n_runs: int, n_ref: int, n_test: int, n_features: int, mu: float = 0.) -> dict:\n", + "def experiment(detector: str, backend: str, n_runs: int, n_ref: int, n_test: int, n_features: int, \n", + " mu: float = 0.) -> dict:\n", " \"\"\" Runs the experiment n_runs times, each time with newly sampled reference and test data.\n", " Returns the p-values for each test as well as the mean and standard deviations of the runtimes. \"\"\"\n", " p_vals, t_detect = [], []\n", @@ -148,28 +135,49 @@ " # Sample reference and test data\n", " x_ref = np.random.randn(*(n_ref, n_features)).astype(np.float32)\n", " x_test = np.random.randn(*(n_test, n_features)).astype(np.float32) + mu\n", - "\n", + " \n", " # Initialise detector, make and log predictions\n", " p_val = .05\n", - " dd = MMDDrift(x_ref, backend=backend, p_val=p_val, n_permutations=100)\n", + " dd_kwargs = dict(p_val=p_val, backend=backend, n_permutations=100)\n", + " if detector == 'mmd':\n", + " dd = MMDDrift(x_ref, **dd_kwargs)\n", + " elif detector == 'learned_kernel':\n", + " d_out, sigma = 2, .1\n", + " proj = Projection(n_features, d_out)\n", + " Kernel = GaussianRBFKeops if backend == 'keops' else GaussianRBFTorch\n", + " kernel_a = Kernel(trainable=True, sigma = torch.Tensor([sigma]))\n", + " kernel_b = Kernel(trainable=True, sigma = torch.Tensor([sigma]))\n", + " device = torch.device('cuda')\n", + " DeepKernel = DeepKernelKeops if backend == 'keops' else DeepKernelTorch\n", + " deep_kernel = DeepKernel(proj, kernel_a, kernel_b, eps=.01).to(device)\n", + " if backend == 'pytorch' and n_ref + n_test > 20000:\n", + " batch_size = 10000\n", + " else:\n", + " batch_size = 1000000\n", + " dd_kwargs.update(\n", + " dict(\n", + " epochs=2, train_size=.75, batch_size=batch_size, batch_size_predict=1000000\n", + " )\n", + " )\n", + " dd = LearnedKernelDrift(x_ref, deep_kernel, **dd_kwargs)\n", " start = timer()\n", " pred = dd.predict(x_test)\n", " end = timer()\n", - "\n", + " \n", " if _ > 0: # first run reserved for KeOps compilation\n", " t_detect.append(end - start)\n", " p_vals.append(pred['data']['p_val'])\n", - "\n", + " \n", " del dd, x_ref, x_test\n", " torch.cuda.empty_cache()\n", - "\n", + " \n", " p_vals = np.array(p_vals)\n", " t_mean, t_std = np.array(t_detect).mean(), np.array(t_detect).std()\n", - " results = eval_detector(p_vals, p_val, mu == 0., t_mean, t_std)\n", + " results = eval_detector(p_vals, p_val, mu != 0., t_mean, t_std)\n", " return results\n", "\n", "\n", - "def format_results(n_features: list, backends: list, max_batch_size: int = 1e10) -> dict:\n", + "def format_results(experiments: dict, n_features: list, backends: list, max_batch_size: int = 1e10) -> dict:\n", " T = {'batch_size': None, 'keops': None, 'pytorch': None}\n", " T['batch_size'] = np.unique([experiments['keops'][_]['n_ref'] for _ in experiments['keops'].keys()])\n", " T['batch_size'] = list(T['batch_size'][T['batch_size'] <= max_batch_size])\n", @@ -189,9 +197,9 @@ " return T\n", "\n", "\n", - "def plot_absolute_time(results: dict, n_features: list, y_scale: str = 'linear',\n", + "def plot_absolute_time(experiments: dict, results: dict, n_features: list, y_scale: str = 'linear', \n", " detector: str = 'MMD', max_batch_size: int = 1e10):\n", - " T = format_results(n_features, ['keops', 'pytorch'], max_batch_size)\n", + " T = format_results(experiments, n_features, ['keops', 'pytorch'], max_batch_size)\n", " colors = ['b', 'g', 'r', 'c', 'm', 'y', 'b']\n", " legend, n_c = [], 0\n", " for f in n_features:\n", @@ -208,9 +216,9 @@ " plt.show();\n", "\n", "\n", - "def plot_relative_time(results: dict, n_features: list, y_scale: str = 'linear',\n", + "def plot_relative_time(experiments: dict, results: dict, n_features: list, y_scale: str = 'linear',\n", " detector: str = 'MMD', max_batch_size: int = 1e10):\n", - " T = format_results(n_features, ['keops', 'pytorch'], max_batch_size)\n", + " T = format_results(experiments, n_features, ['keops', 'pytorch'], max_batch_size)\n", " colors = ['b', 'g', 'r', 'c', 'm', 'y', 'b']\n", " legend, n_c = [], 0\n", " for f in n_features:\n", @@ -229,16 +237,13 @@ }, { "cell_type": "markdown", - "id": "43a4ee7e", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ - "As detailed earlier, we will compare the PyTorch with the KeOps implementation of the MMD detector for a variety of reference and test data batch sizes as well as different feature dimensions. Note that for the PyTorch implementation, the portion of the kernel matrix for the reference data itself can already be computed at initialisation of the detector. This computation will not be included when we record the detector's prediction time. Since use cases where $N_\\text{ref} \\gg N_\\text{test}$ are quite common, we will also test for this specific setting. The key reason is that we cannot amortise this computation for the KeOps detector since we are working with lazily evaluated symbolic matrices.\n", + "As detailed earlier, we will compare the PyTorch with the KeOps implementation of the MMD and learned kernel MMD detectors for a variety of reference and test data batch sizes as well as different feature dimensions. Note that for the PyTorch implementation, the portion of the kernel matrix for the reference data itself can already be computed at initialisation of the detector. This computation will not be included when we record the detector's prediction time. Since use cases where $N_\\text{ref} >> N_\\text{test}$ are quite common, we will also test for this specific setting. The key reason is that we cannot amortise this computation for the KeOps detector since we are working with lazily evaluated symbolic matrices.\n", "\n", - "#### $N_\\text{ref} = N_\\text{test}$\n", + "### MMD detector\n", + "\n", + "#### 1. $N_\\text{ref} = N_\\text{test}$\n", "\n", "Note that for KeOps we could further increase the number of instances in the reference and test sets (e.g. to 500,000) without running into memory issues." ] @@ -246,45 +251,40 @@ { "cell_type": "code", "execution_count": 3, - "id": "47268603", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "metadata": {}, "outputs": [], "source": [ - "experiments = {\n", + "experiments_eq = {\n", " 'keops': {\n", - " 0: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 10, 'n_features': 2},\n", - " 1: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 10, 'n_features': 2},\n", - " 2: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 10, 'n_features': 2},\n", - " 3: {'n_ref': 20000, 'n_test': 20000, 'n_runs': 10, 'n_features': 2},\n", - " 4: {'n_ref': 50000, 'n_test': 50000, 'n_runs': 10, 'n_features': 2},\n", - " 5: {'n_ref': 100000, 'n_test': 100000, 'n_runs': 10, 'n_features': 2},\n", - " 6: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 10, 'n_features': 10},\n", - " 7: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 10, 'n_features': 10},\n", - " 8: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 10, 'n_features': 10},\n", - " 9: {'n_ref': 20000, 'n_test': 20000, 'n_runs': 10, 'n_features': 10},\n", - " 10: {'n_ref': 50000, 'n_test': 50000, 'n_runs': 10, 'n_features': 10},\n", - " 11: {'n_ref': 100000, 'n_test': 100000, 'n_runs': 10, 'n_features': 10},\n", - " 12: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 10, 'n_features': 50},\n", - " 13: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 10, 'n_features': 50},\n", - " 14: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 10, 'n_features': 50},\n", - " 15: {'n_ref': 20000, 'n_test': 20000, 'n_runs': 10, 'n_features': 50},\n", - " 16: {'n_ref': 50000, 'n_test': 50000, 'n_runs': 10, 'n_features': 50},\n", - " 17: {'n_ref': 100000, 'n_test': 100000, 'n_runs': 10, 'n_features': 50}\n", + " 0: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 5, 'n_features': 2},\n", + " 1: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 5, 'n_features': 2},\n", + " 2: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 5, 'n_features': 2},\n", + " 3: {'n_ref': 20000, 'n_test': 20000, 'n_runs': 5, 'n_features': 2},\n", + " 4: {'n_ref': 50000, 'n_test': 50000, 'n_runs': 5, 'n_features': 2},\n", + " 5: {'n_ref': 100000, 'n_test': 100000, 'n_runs': 5, 'n_features': 2},\n", + " 6: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 5, 'n_features': 10},\n", + " 7: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 5, 'n_features': 10},\n", + " 8: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 5, 'n_features': 10},\n", + " 9: {'n_ref': 20000, 'n_test': 20000, 'n_runs': 5, 'n_features': 10},\n", + " 10: {'n_ref': 50000, 'n_test': 50000, 'n_runs': 5, 'n_features': 10},\n", + " 11: {'n_ref': 100000, 'n_test': 100000, 'n_runs': 5, 'n_features': 10},\n", + " 12: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 5, 'n_features': 50},\n", + " 13: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 5, 'n_features': 50},\n", + " 14: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 5, 'n_features': 50},\n", + " 15: {'n_ref': 20000, 'n_test': 20000, 'n_runs': 5, 'n_features': 50},\n", + " 16: {'n_ref': 50000, 'n_test': 50000, 'n_runs': 5, 'n_features': 50},\n", + " 17: {'n_ref': 100000, 'n_test': 100000, 'n_runs': 5, 'n_features': 50}\n", " },\n", " 'pytorch': { # runs OOM after 10k instances in ref and test sets\n", - " 0: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 10, 'n_features': 2},\n", - " 1: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 10, 'n_features': 2},\n", - " 2: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 10, 'n_features': 2},\n", - " 3: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 10, 'n_features': 10},\n", - " 4: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 10, 'n_features': 10},\n", - " 5: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 10, 'n_features': 10},\n", - " 6: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 10, 'n_features': 50},\n", - " 7: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 10, 'n_features': 50},\n", - " 8: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 10, 'n_features': 50}\n", + " 0: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 5, 'n_features': 2},\n", + " 1: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 5, 'n_features': 2},\n", + " 2: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 5, 'n_features': 2},\n", + " 3: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 5, 'n_features': 10},\n", + " 4: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 5, 'n_features': 10},\n", + " 5: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 5, 'n_features': 10},\n", + " 6: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 5, 'n_features': 50},\n", + " 7: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 5, 'n_features': 50},\n", + " 8: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 5, 'n_features': 50}\n", " }\n", "}" ] @@ -292,12 +292,8 @@ { "cell_type": "code", "execution_count": 4, - "id": "d556296a", "metadata": { - "scrolled": true, - "pycharm": { - "name": "#%%\n" - } + "scrolled": true }, "outputs": [], "source": [ @@ -305,21 +301,16 @@ "results = {backend: {} for backend in backends}\n", "\n", "for backend in backends:\n", - " exps = experiments[backend]\n", + " exps = experiments_eq[backend]\n", " for i, exp in exps.items():\n", " results[backend][i] = experiment(\n", - " backend, exp['n_runs'], exp['n_ref'], exp['n_test'], exp['n_features']\n", + " 'mmd', backend, exp['n_runs'], exp['n_ref'], exp['n_test'], exp['n_features']\n", " )" ] }, { "cell_type": "markdown", - "id": "93396443", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ "Below we visualise the runtimes of the different experiments. We can make the following observations:\n", "\n", @@ -335,16 +326,11 @@ { "cell_type": "code", "execution_count": 5, - "id": "5d854bfb", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAEWCAYAAAAQHy/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdeVyVVf7A8c9hRwEVQYIUrybLRcxcsrQ0DWUcRdMcydFwLbXyl2mbM9OijVMzU5jpVC5l5pSNaZmO2qZh6Vg52qLIMi7hkoigoriy3PP743mwK4Iscbks3/frdV/c+6zfe7lwvs855zlHaa0RQgghRMPi4uwAhBBCCFHzJAEQQgghGiBJAIQQQogGSBIAIYQQogGSBEAIIYRogCQBEEIIIRogSQDqCKWUVkq1u8b6BUqpp+1eP6CUylJKnVVKNa/C+SzmOd2qGnN1Md9D2xo4T6h5LtcaOFeQUuorpVSeUirR0ecTFaeU+qNS6g1nxyGEo0kCUIJSKkMpla+UCiix/HuzQLSYr5ear+8qsd3L5vKx5uuxSqkis2A5q5T6SSn1llIqvDrj1lpP1lr/2TynOzAHiNVa+2itT5SXQPwa5nvcWk3H2qyUus9+mfkeDlTH8UucK0Mp1dfuPIfMcxVV97lKMRHIAfy01o/+2oMppYKVUmuVUkftv6d26z2VUkuUUmeUUseUUtNLrI9RSqUppc4rpZKUUq1/bUzOoJSaqZR6pxLb91ZKHbFfprV+Xmt9X1n7CFFfSAJQup+A3xe/UEp1ABqVst3/gNF227kB8cD+Ett9rbX2AZoAfYELwE6lVHR1BFvKFWsQ4AXsqY7jC4doDaToKozEVUatjA34BBhWxm4zgTDzvH2AJ5RS/c3jBQAfAk8D/sAOYEVl46qs2lC7JESDprWWh90DyACeAv5rt+wl4E+ABizmsqXm8iygmbksDvgY2AqMNZeNBbaWcp51wKprxPE4kAkcBcab525nd+7XgQ3AOYykYikwGwg3l2ngLPAF8JX5+py57J5Szudqvp8c4ADwkLmPm7m+CfCmGdPP5rlcAStwESgyj51rbu9pHu+Q+RktALztzncX8ANwBiNh6g/8xTzORfNY/zC3tX/vTYBlQDZw0Pxdudh/1uZ5T2Ekcr8t4/P9J0ahecE81xOApcR73my+z23mNv8GmgPvmnH/t/j7YG4fCXwOnATSgfgyzr0UKADyzeP2NT+vuebv+6j53NPcvjdwBHgSOAb88xrfGzfsvqd2y49i1AgVv/4z8C/z+URgm926xubnEnmNv5E/ACnm5/wW4GW3Ps783eaan92NJfZ9EtgFXDLjzcD4vu/C+I6+iZHEfgzkARv55W+sN3CklHj6YnyH8s3P9izwo7l+HJBqHusAMKnE+7SZ258FQjCSpXfsjj8YI5nONb8T1hLnfsyM/TRG4uRlrgvA+DvPNb8TWzC/q/KQR214OD2A2vaw+2eSjlG4uZr/fFtzdQIwG1gEPGAuex+j5qAiCcB4IKuMGPpjFJrR5j+p5VydAJwGbsOoxfEqjsdcb8GuIDOXXd6/jHNOBtKAVhhXgUlcWRiuBhaa8bQAttv9I73qPQIvA2vNY/liFJ4vmOu6mfH3M+O/HrOwMf/B3lfiWPbvfRmwxjymBaMWZoJdHAXA/ebv7QGMgk9d63dt9/qKz82MZR9wA0bikWKery9GwbUMeMvctjFwGKOwcQM6YSRTUWWc+/Lvy3z9HPCN+dkGYhScfzbX9QYKgb9hJArepR3T3PaqBABoZi4Lslv2O2C3+fwV4PUSx0kGhl3jc0u2+678h1++e52A48At5u9gjLm9p92+P5j7etst+waj0L/e3P8781heGEnss3afRakJgPl8JnaFt7lsoPk7VMAdwHmg8zWOd/kY/JJQ9wPcMRLFfYCH3bm3YyQO/hiJxmRz3QsYia+7+ehJGd9FecjDGQ9pAijbPzGq9/th/FH/XMZ2y4DRSqmmGP9cPqrg8Y9i/MMoTTxGwZKstT6H8Q+ppDVa6/9orW1a64sVPOe1xANztdaHtdYnMf55AUaHNWAA8IjW+pzW+jhGAT+itAMppRTGVeU0rfVJrXUe8Lzd9hOAJVrrz834f9Zap5UXoNnUMQL4g9Y6T2udASQCCXabHdRaL9ZGO/7bQDBGwVJVb2mt92utT2Ncke7XWm/UWhcCKzEKKTCuejO01m9prQu11t8DHwDDK3ieUcBzWuvjWutsYFaJ92XDKAQvaa0vVPI9+Jg/T9stO42RRBWvP82V7NeX5h9235W/8EuT2URgodb6W611kdb6bYwr/Vvt9p1n7mv/PuZrrbO01j9jXCl/q7X+3vxur+aXz7nStNbrzd+h1lp/CXyGURhXxD3AevO7WoBRu+QN9Cjxfo6an8W/gZvM5QUY37/WWusCrfUWrbVMviJqDWmDK9s/MarO22AU8qXSWm9VSgViNBGs01pfMMq/cl2PUS1YmhBgp93rg6Vsc7giJ6mEkBLHtD9na4wrmEy79+ZyjRgCMfpM7LTbXmFcEYJx9behCjEGmHHYx3YQ47Msdqz4idb6vHl+H6ouy+75hVJeFx+7NXCLUirXbr0bxveoIkK4+n2F2L3O/hWJ3lnzpx9G80rx8zy79X4l9rFfX5qS35XiWFsDY5RS/2e33oMr30tp35uKfs6VppT6LfAsxtW8C8Z3c3cFd7/i96K1timlDlPGdw6jdqH4vb6Ikbx/Zn4PF2mt/1qFtyCEQ0gNQBm01gcx2pAHYHSQupZ3gEe5RqJQiqEYVzqlycQoJIuFlhZiJc5VEdc652GMq7gArXVT8+GntW5fRiw5GP+029tt30QbHSGLj3dDGXFc633lYFxV2fdQD6Xs2pnyVOdneBj40u79NtXGHQUPVHD/o1z9vo7ava5yrFrrUxi/3452izvySyfRPfbrlFKNMX4/1+pEWvK7UhzrYeAvJT6HRlrr9+xDqto7AYzq+Msdcs1aocCyjq2U8sSoiXkJowmkKUbyqUrbvhRX/F7M2q1WVOA7Z9ZSPaq1bovRj2C6UiqmvP2EqCmSAFzbBOBOsxr+WuZhNBV8da2NlFKuSqk2Sqn5GG2Ps8rY9H1grFIqSinVCOPq5dfKAq51L/37wMNKqZZKqWbAjOIVWutMjGrTRKWUn1LKRSl1g1LqDrtjt1RKeZjb24DFwMtKqRYASqnrlVK/Mbd/Exhn3nrmYq6LLC9Os1r/feAvSilf81a16RgJWFWU95lUxjogXCmVoJRyNx83K6WsFdz/PeAppVSg2Sv/GSr5vpRSXhh9BAA8zdfFlpnHb2Z+1vdj9EMAo4o9Wik1zNznGWBXOc0yD5nfFX+M2q/iuwYWA5OVUrcoQ2Ol1ECl1LWaEyrjf4CXeUx3jE6gnnbrswCLUqr4f5uHuT4bKDRrA2JLbN9cKdWkjPO9Dww0v6vuGIn+JYw+GteklIpTSrUzk4bTGB1cbRV9o0I4miQA12C2G+6owHYntdabrtG+110pdRaj5/hmjOrVm7XWpVZDaq0/xugF/gVGh6MvqhJ/CTOBt5VSuUqp+FLWLwY+BX7E6IBVstZjNMY/0+Ke36sw2jcx49sDHFNK5ZjLnjRj/0YpdQajJ3eE+f62Y3SWexnjH+OX/HKV9QrwO6XUKaXUvFLi/D+Mq8ADGJ0tlwNLKvYRXOUFjEIxVyn1WBWPARhXexgFywiMq8Zj/NJpryJmY9x+twujevo7c1llFN/RAEaHTvs29mcx7rY4iPF5v6i1/sSMPRvj9sG/YPxub6GM/h12lmMkhQfM4842j7UDI7n4h3msfRidM6uF2RfjQeANjKvwcxiddIutNH+eUEp9Z/5eHsYoyE8BIzE6pxYfLw0j+Tpgfg/smyrQWqcD9wLzMWqgBgGDtNb5FQg3DON7fxb4GnhNa51UuXcshOMo6ZMihKgMpVQGxp0aG50dixCi6qQGQAghhGiAJAEQQgghGiBpAhBCCCEaIKkBEEIIIRqgOjEQUEBAgLZYLM4OQwgh6pSdO3fmaK0Dy99SNER1IgGwWCzs2FHu3XhCCCHsKKVKG0VUCECaAIQQQogGSRIAIYQQogGSBEAIIYRogCQBEEIIIRogSQCEEEKIBkgSACGEEKIBkgRACCGEaIAcmgAopaYppfYopZKVUu8ppbyUUm2UUt8qpfYppVYUzyEvhBDCTlYWPPIIXLrk7EhEPeWwBEApdT3GPNxdtdbRgCvGHON/A17WWrfDmJ97gqNiEEKIOunIEejVCxYvhj17nB2NqKcc3QTgBngrpdyARkAmcCewylz/NjDEwTEIIUTdsX8/9OwJx47Bp59C587OjkjUUw5LALTWPwMvAYcwCv7TwE4gV2tdaG52BLi+tP2VUhOVUjuUUjuys7MdFaYQQtQeqanGlf+ZM/DFF3D77c6OSNRjjmwCaAbcBbQBQoDGQP+K7q+1XqS17qq17hoYKHNZCCHque+/Nwp/mw2+/BK6dHF2RKKec2QTQF/gJ611tta6APgQuA1oajYJALQEfnZgDEIIUft9/TX06QPe3vDVVxAd7eyIRAPgyATgEHCrUqqRUkoBMUAKkAT8ztxmDLDGgTEIIUTt9sUX0K8fBAbC1q0QFubsiEQD4cg+AN9idPb7DthtnmsR8CQwXSm1D2gOvOmoGIQQolZbvx4GDACLxbjyDw11dkSiAXErf5Oq01o/CzxbYvEBoJsjzyuEELXeypUwciR07Gj09m/e3NkRiQZGRgIUQoiatnQpjBgBt94KmzZJ4S+cQhIAIYSoSa++CuPGQUwMfPIJNGni7IhEAyUJgBBC1JS//Q2mTIHBg2HtWmjc2NkRiQZMEgAhhHA0reHpp2HGDKPqf9Uq8PJydlSigXNoJ0AhhGjwtIZHH4WXX4YJE2DhQnB1dXZUQkgNgBBCOExREUyaZBT+Dz8MixZJ4S9qDUkAhBDCEQoKYPRoY0a/P/0J5s4FF/mXK2oPaQIQQojqdukS3HMPrFkDL7xgtP0LUctIAiCEENXp/HkYOhQ++wzmzzd6/QtRC0kCIIQQ1eXMGYiLg//8B5YsMe73F6KWkgRACCGqw4kT0L8//PADvPcexMc7OyIhrkkSACGE+LWOHTNm9Nu7F1avNmoBhKjlJAEQQohf49Ah6NsXjh41ZveLiXF2REJUiCQAQghRVfv2GQX+6dNGp78ePZwdkRAVJgmAEEJUxZ49xpV/QQF88QV07uzsiISoFIeNSqGUilBK/WD3OKOUekQp5a+U+lwptdf82cxRMQghhEPs3Al33AFKwVdfSeEv6iSHJQBa63St9U1a65uALsB5YDUwA9iktQ4DNpmvhRCibvjPf+DOO8HHB7ZsgagoZ0ckRJXU1LiUMcB+rfVB4C7gbXP528CQGopBCCF+nY0bITYWrrvOKPxvuMHZEQlRZTWVAIwA3jOfB2mtM83nx4Cg0nZQSk1USu1QSu3Izs6uiRiFEKJs//43DBxoFPpffQWtWjk7IiF+FYcnAEopD2AwsLLkOq21BnRp+2mtF2mtu2qtuwYGBjo4SiGEuIYVK+Duu6FjR9i8GYJKvW4Rok6piRqA3wLfaa2zzNdZSqlgAPPn8RqIQQghqmbJEvj9741b/DZuBH9/Z0ckRLWoiQTg9/xS/Q+wFhhjPh8DrKmBGIQQovLmzYMJE4x2/48/Bj8/Z0ckRLVxaAKglGoM9AM+tFv8V6CfUmov0Nd8LYQQtcvzz8PUqTBkiDGtb6NGzo5IiGrl0IGAtNbngOYllp3AuCtACCFqH63hT3+CF16AkSNh6VJwd3d2VEJUOxkJUAghitlsMG2aUfV///3w+uvg6ursqIRwCEkAhBACoKgIJk40Ov1NmwaJicZIf0LUUzU1DoAQQtReBQUwapRR+D/zjBT+okGQGgAhRMN28SLExxsD/fz97/D4486OSIgaIQmAEKLhOncO7roLNm2CV1+FBx90dkRC1BhJAIQQDdPp08bQvl9/bfT0HzOm3F2EqE8kARBCNDw5OfCb38Du3cYwv7/7nbMjEqLGSQIghGhYMjOhb184cAA++ggGDHB2REI4hSQAQoiG4+BBiImBY8dgwwbo08fZEQnhNJIACCEahv/9z7jyz8szJvW59VZnRySEU0kCIISo/3bvhn79jJH+kpLgppucHZEQTicDAQkh6rcdO6B3b2NI36++ksJfCJMkAEKI+mvLFrjzTmjSxHgeGensiISoNSQBEELUT599ZtzqFxJiXPm3bevsiISoVSQBEELUPx99BIMGQXi4Ufi3bOnsiISodSQBEELUL8uXGwP7dOpkdPhr0cLZEQlRKzk0AVBKNVVKrVJKpSmlUpVS3ZVS/kqpz5VSe82fzRwZgxCiAVm8GO69F3r2hM8/h2by70WIsji6BuAV4BOtdSTQEUgFZgCbtNZhwCbztRBC/Dpz58LEidC/vzHIj6+vsyMSolZzWAKglGoC9ALeBNBa52utc4G7gLfNzd4GhjgqBiFEA6A1zJ4N06bBsGFG+7+3t7OjEqLWc2QNQBsgG3hLKfW9UuoNpVRjIEhrnWlucwwIKm1npdREpdQOpdSO7OxsB4YphKiztIYZM+DppyEhAf71L/DwcHZUQtQJjkwA3IDOwOta607AOUpU92utNaBL21lrvUhr3VVr3TUwMNCBYQoh6iSbDaZMgb//HSZPNqb0dZPBTYWoKEcmAEeAI1rrb83XqzASgiylVDCA+fO4A2MQQtRHhYUwfjy89ho89pjx00VuahKiMhz2F6O1PgYcVkpFmItigBRgLTDGXDYGWOOoGIQQ9VB+PowcCW+/DbNmGTUASjk7KiHqHEfXl/0f8K5SygM4AIzDSDreV0pNAA4C8Q6OQQhRX1y4AMOHw/r18NJL8Oijzo5IiDrLoQmA1voHoGspq2IceV4hRD109iwMHgybN8OCBTBpkrMjEqJOkx4zQojaLzcXBgyA7dth2TJjsB8hxK8iCYAQonbLzobYWNizB1auhKFDnR2REPWCJABCiNrr55+hXz/46SdYu9YY5U8IUS0kARBC1E4ZGRATA8ePwyefwB13ODsiIeoVSQCEELVPerpR+J8/D5s2Qbduzo5IiHpHEgAhRO2ya5dR7Q9Gj/8bb3RqOELUVzJ0lhCi9ti+HXr3Nsbz/+orKfyFcCBJAIQQtcOXXxrV/s2awZYtEBFR/j5CiCqTBEAI4XyffGL08G/Vyij8LRZnRyREvScJgBDCuT780Bjhz2o1agFCQpwdkRANgiQAQgjneecdiI+Hrl3hiy9Apv4WosZIAiCEcI6FC2H0aOP+/s8+g6ZNnR2REA2KJABCiJqXmAiTJxvj+69fDz4+zo5IiAZHEgAhRM3RGmbNgsceM6b1/fBD8PJydlRCNEgyEJAQomZoDY8/blz9jx0Lb7wBrq7OjkqIBsuhCYBSKgPIA4qAQq11V6WUP7ACsAAZQLzW+pQj4xBCOJnNBg89BAsWGD/nzQMXqYAUwplq4i+wj9b6Jq11V/P1DGCT1joM2GS+FkLUV4WFxhX/ggXw5JMwf74U/kLUAs74K7wLeNt8/jYwxAkxCCFqQn4+3HMP/POfMHs2vPACKOXsqIQQOD4B0MBnSqmdSqmJ5rIgrXWm+fwYEFTajkqpiUqpHUqpHdnZ2Q4OUwhR7S5cgCFDjI5+L78Mf/qTFP5C1CKO7gR4u9b6Z6VUC+BzpVSa/UqttVZK6dJ21FovAhYBdO3atdRthBC1VF4eDBpkTOizeDHcd5+zIxJClODQGgCt9c/mz+PAaqAbkKWUCgYwfx53ZAxCiBp26pQxne/WrfDuu1L4C1FLOSwBUEo1Vkr5Fj8HYoFkYC0wxtxsDLDGUTEIIWrY8ePQpw98/z188AH8/vfOjkgIUQZHNgEEAauV0ebnBizXWn+ilPov8L5SagJwEIh3YAxCiJpy5Aj07QuHDsG6dUYtgBCi1nJYAqC1PgB0LGX5CSDGUecVQjjBgQMQEwMnTsCnn0LPns6OSAhRjnITAKVUd+BeoCcQDFzAqMpfD7yjtT7t0AiFELVbaqpx5X/xojGjX9eu5e8jhHC6a/YBUEp9DNwHfAr0x0gAooCnAC9gjVJqsKODFELUUj/8AL16QVERfPmlFP5C1CHl1QAkaK1zSiw7C3xnPhKVUgEOiUwIUbt98w389rfg6wubNkFYmLMjEkJUwjVrAIoLf7NHv4v5PFwpNVgp5W6/jRCiAUlKMqr9AwJgyxYp/IWogyp6G+BXgJdS6nrgMyABWOqooIQQtdiGDTBgAFgsxkA/rVs7OyIhRBVUNAFQWuvzwN3Aa1rr4UB7x4UlhKiVVq0yhveNioLNmyE42NkRCSGqqMIJgHk3wCiM3v8AMpG3EA3J228bE/t062b09g+Q7j9C1GUVTQCmAn8AVmut9yil2gJJjgtLCFGrvPaaMaXvnXca9/k3aeLsiIQQv1KFBgLSWn+F0Q+g+PUB4GFHBSWEqEX+/nd48kkYPBhWrAAvL2dHJISoBuWNA7BYKdWhjHWNlVLjlVKjHBOaEMKptIZnnjEK/xEjjPZ/KfyFqDfKqwF4FXjaTAKSgWyMAYDCAD9gCfCuQyMUQtQ8reHRR+Hll2HCBFi4EFyl248Q9ck1EwCt9Q9AvFLKB+jKL0MBp2qt02sgPiFETSsqggcegMWL4eGHjSTAxaEzhwshnKCifQDOApsdG4oQwukKCozOfsuXwx//CLNngzGjpxBVsnPnzhZubm5vANE4cAp6USobkFxYWHhfly5djpdc6cjpgIUQdcmlS8ZtfmvWwPPPwx/+4OyIRD3g5ub2xnXXXWcNDAw85eLiop0dT0Nis9lUdnZ21LFjx94Arpq3R7IxIQScP2/08l+zBubNk8JfVKfowMDAM1L41zwXFxcdGBh4GqP25SqVqgFQSjUyRwQUQtQXZ85AXBz85z/w5pswfryzIxL1i4sU/s5jfvalXuxXqAZAKdVDKZUCpJmvOyqlXqvgvq5Kqe+VUuvM122UUt8qpfYppVYopTwq9jaEENXu5EljUp+vvzba/aXwF6LBqGgTwMvAb4ATAFrrH4FeFdx3KpBq9/pvwMta63bAKWBCBY8jhKhOWVnQuzfs2gUffmi0/wtRz6Snp3uEhYXV2rlrBg8e3MZisUSHhYW1Hz58uOXSpUs11uu2wn0AtNaHSywqKm8fpVRLYCDwhvlaAXcCq8xN3gaGVDQGIUQ1OXwYevaE/fth/XoYNMjZEQnRII0aNerkgQMHktPT0/dcvHhRzZ07t8Ym2ahoAnBYKdUD0Eopd6XUY1x5VV+WucATGLciADQHcrXWhebrI8D1pe2olJqolNqhlNqRnZ1dwTCFEOXat88o/LOy4LPPICbG2REJUSNSUlI8rFZr1JdfftmosLCQSZMmtYyOjraGh4dHvfjiiwEANpuNSZMmtQwLC2sfHh4etXjx4mYA69at8+3atWtE796921ksluiRI0eGFhUVUVhYyLBhwyzF28+aNatFZWK65557Tru4uODi4kLXrl3PHTlypMaaxSvaCXAy8ApGYf0z8Bnw0LV2UErFAce11juVUr0rG5jWehGwCKBr167SgUSI6pCSYrT55+dDUhJ07uzsiEQDMn48rZKTaVSdx4yO5vySJZSsob7Kjz/+6DlixIgblixZ8lP37t0vvPTSSwFNmjQpSk5OTr1w4YK6+eabIwcNGnTmm2++abR7927v1NTUPZmZmW7dunWzxsbGngXYvXt34++//z45PDw8v1evXmHLli1r1q5du0uZmZnue/fu3QOQk5NTpSEzL126pFasWNF8zpw55b6X6lLRgYByMKYCrozbgMFKqQEYwwf7YSQRTZVSbmYtQEuMhEII4WjffQexseDhAV9+Ce1rbbOoENXq5MmTbkOGDGm3atWq/V26dLkIsHHjRr+0tLRGa9eubQaQl5fnmpKS4rVlyxbf+Pj4k25ubrRq1arwlltuObt169ZGTZo0sXXo0OFcVFRUPkB8fPzJLVu2+MTFxZ05fPiw55gxY1oNGjTo9NChQ89UJcYxY8aE3nrrrWf79+9/tvre+bVVKAFQSrUB/g+w2O+jtb5qYAG7dX/AmEIYswbgMa31KKXUSuB3wL+AMcCaKsYuhKiobdvgt7+Fpk1h0yZo187ZEYkGqCJX6o7g6+tbFBISkp+UlORTnABorVViYuKhYcOGXVFgr1+/vsy5rlWJUTGVUgQGBhYlJyenrF692m/BggWBK1as8F+5cmVG8TaFhYVER0dHAfTv3z937ty5R0se99FHHw3Oyclx+/TTT/f/undaORXtA/ARkAHMBxLtHlXxJDBdKbUPo0/Am1U8jhCiIjZtgn79ICgItm6Vwl80OO7u7vrjjz/e/9577zVfsGCBP0C/fv1Ov/7664HFve537drleebMGZdevXrlrVq1yr+wsJCjR4+6bd++3adnz57nwGgCSEtL8ygqKmLVqlX+PXv2zMvMzHQrKipi7NixuS+88MLPu3fvvqKJw83NjbS0tJS0tLSU0gr/OXPmBHzxxRdNPvroowOuNTzhVkX7AFzUWs+r6km01psx5xLQWh8AulX1WEKISvj3v2H4cAgLg88/h+uuc3ZEQjiFn5+f7dNPP93Xu3fvcF9f36Jp06blZGRkeHbo0MGqtVb+/v4FGzZs2J+QkJC7bds2H6vV2l4ppWfNmnUkNDS0cNeuXURHR5+bPHlyaEZGhlePHj3OJCQk5G7fvt17woQJFpvNpgCee+65I5WJ64knnmgdHBx8qWvXrlaAuLi4Uy+99FKmIz6DkpTW5fevU0qNxJgC+DPgUvFyrfV3jgvtF127dtU7duyoiVMJUX+sWAH33gs33QSffALNmzs7IlHDlFI7tdZdnRnDjz/+mNGxY8ccZ8ZQHdatW+ebmJgYlJSUtM/ZsVTWjz/+GNCxY0dLyeUVrQHoACRg3MNffEufNl8LIWqbt96C++6D226DdevAz8/ZEQkhapmKJgDDgbZa63xHBiOEqAbz58PDDxs9/levhkbVeteVEA1SXFxcXlxcXJ6z46hOFe0EmAw0dWQgQohq8MILRuE/ZAisXSuFvxCiTBWtARjljYkAACAASURBVGgKpCml/suVfQDKvA1QCFGDtIannoLnn4eRI2HpUnB3d3ZUQoharKIJwLMOjUIIUXU2G0ybBvPmwf33w+uvQw3fTiSEqHsqOhLgl44ORAhRBUVFMHEiLFkCjzwCc+aAqrHJxIQQddg1+wAopbaaP/OUUmfsHnlKqSoNdyiEqCYFBTBqlFH4P/20FP5CVJMZM2ZU24AZ06dPD3nmmWeCqrr/tm3bvG+66abIdu3aXTE5UXUorxNgYwCtta/W2s/u4au1lvuKhHCWixdh2DDjXv+//Q2ee04KfyGqybx584Irs73NZqOoqMghsfj4+Nj++c9//rRv3749n3322d4//vGPrao64VBJ5SUAMgufELXNuXMQF2eM8vfqq/DEE86OSIhaKz093aNNmzbtBw8e3KZt27bt+/fv3zYvL89l7dq1vn379r2heLvVq1f79evX74YHH3zw+kuXLrlERkZGDR48uA3AzJkzg8LCwtqHhYW1f+6551oUH9disUQPHTrUEh4e3n7//v0eq1at8ouKirJGREREde/ePbz42Kmpqd7dunWLaNmyZYfZs2dXarrgG2+88VKHDh0uAVgslgJ/f//CzMzMivbfu6byDtJCKTW9rJVa6znVEYQQooJOn4aBA+Hrr42e/mPGODsiISqlWzciSi67+25OzphBdl4eLjExhJVcf++95Dz8MCcyM3G76y5usF+3fTvp5Z0zIyPDa+HChRmxsbHnhg8fbnnxxRcDZ86cmTV16tTQo0ePuoWEhBQuWbKk+bhx43JGjhx5eunSpS3S0tJSALZs2dJo+fLlzXfu3JmqtaZLly7WmJiYvICAgKJDhw55vvnmmz/FxMRkHD161G3KlCmWzZs3p0VGRuZnZWVdvkrft2+f17Zt29Jzc3NdrVZr9OOPP57t6elZ6QvspKSkRgUFBSoqKupS+VuXr7waAFfAB/At4yGEqCk5ORATA99+C//6lxT+QlTQddddlx8bG3sOICEh4cS2bdt8XFxciI+PP7F48WL/nJwc1++++85n+PDhp0vuu3nzZp8BAwbk+vn52Zo0aWIbOHDgqaSkJF+A4ODg/JiYmHPmdo27deuWFxkZmQ8QFBR0uU0gNjY219vbWwcHBxf6+/sXHDlypNJX8AcPHnQfN25c28WLF2dU16RB5QWRqbV+rlrOJISousxMY0a/ffvgo4+MWgAh6qBrXbH7+mK71vrgYAorcsVfUmnT+AI88MADJwYOHNjOy8tLDxo06JR7JcfOaNSoka38rcD+at/V1ZXCwsIrAlq2bFnT559/PgRg0aJFGb169Tpvv/7kyZMuv/3tb9s9++yzPxcnHNWhvBoA6VUkhLMdPAi9ekFGBnz8sRT+QlRSZmamx8aNGxsDvPvuu/49evQ4C0abelBQUEFiYmLwxIkTL09Y5ObmpounCe7Tp8/ZDRs2NM3Ly3M5c+aMy4YNG5r16dPnqiGBe/fufW779u2+aWlpHgD2TQDlGT16dG7xlMElC/+LFy+qgQMHthsxYsSJcePGnaraJ1C68hKAmOo8mRCikvbuhZ49ITvbmM63Tx9nRyREnWOxWC7Onz+/Rdu2bdvn5ua6PfbYY9nF60aMGHEiODg4v3PnzheLl40aNSrbarVGDR48uM3tt99+fuTIkSc6d+5s7dKlizUhISH7tttuu1DyHCEhIYXz5s3LGDp0aLuIiIiooUOHtq2O2JcsWdLsv//9r8/y5csDIiMjoyIjI6O2bdvmXR3HrtB0wFU6sFJewFeAJ0ZTwyqt9bNKqTbAv4DmwE4gobxJhmQ6YNEgJSdD377GYD+ffQadOjk7IlHHyHTARm/9uLi4sL179+4pbf3o0aNDO3XqdH7atGl1fsrispQ1HXBFJwOqikvAnVrrjsBNQH+l1K3A34CXtdbtgFPABAfGIETdtGMH3HGHMaTvV19J4S+EA7Rv396akpLiPXny5BPOjsUZquVewtJoo2rhrPnS3Xxo4E5gpLn8bWAm8Lqj4hCiztm6FQYMgObNYdMmaFstNYlCNEgRERH5ZV3979mzJ7Wm46lNHFkDgFLKVSn1A3Ac+BzYD+RqrQvNTY4A15ex70Sl1A6l1I7s7OzSNhGi/vn8c4iNhZAQ2LJFCn8hhMM4NAHQWhdprW8CWgLdgMhK7LtIa91Va901MDDQYTEKUWusWWOM8BcWZlT7t2zp7IiEEPWYQxOAYlrrXCAJ6A40VUoVNz20BH6uiRiEqNXee88Y2/+mmyApCVpUarRQIYSoNIclAEqpQKVUU/O5N9APSMVIBH5nbjYGWOOoGISoE954w5jV7/bbYeNG8Pd3dkRCiAbAkTUAwUCSUmoX8F/gc631OuBJYLpSah/GrYBvOjAGIWq3uXPh/vvhN7+BDRvAV0bYFqI6paene4SFhbV3dhxlef755wNDQ0OjlVJd7Cf5sdlsjB07tlVoaGh0eHh41NatWxtV97kdeRfALuCqe5e01gcw+gMI0XBpDc8/D089BXffDcuXg6ens6MSQtSwO+644+ywYcNO33nnnVdMkrRy5comBw4c8MrIyEhOSkpq/OCDD4bu2rUrrTrPXSN9AIQQdrSGP/zBKPwTEmDFCin8hagBKSkpHlarNerLL79sVFhYyKRJk1pGR0dbw8PDo1588cUAMK68J02a1DIsLKx9eHh41OLFi5sBrFu3zrdr164RvXv3bmexWKJHjhwZWlRURGFhIcOGDbMUbz9r1qxKdeC57bbbLkRERFw1GN6aNWuajho16oSLiwsxMTHnzpw543bw4MHKTVZQDofVAAghSmGzwcMPw6uvwuTJxk8XycNFwzB+zfhWyceTq7UqO7pF9Pkldy05XN52P/74o+eIESNuWLJkyU/du3e/8NJLLwU0adKkKDk5OfXChQvq5ptvjhw0aNCZb775ptHu3bu9U1NT92RmZrp169bNGhsbexZg9+7djb///vvk8PDw/F69eoUtW7asWbt27S5lZma6F481kJOTUy1T9WVmZrpbLJbLiUFwcHD+wYMH3Vu3bl1QHccHqQEQouYUFsKECUah/+ij8NprUvgLUQNOnjzpNmTIkHbvvPPOge7du18A2Lhxo9/777/fPDIyMqpTp07WU6dOuaWkpHht2bLFNz4+/qSbmxutWrUqvOWWW84Wt7936NDhXFRUVL6bmxvx8fEnt2zZ4hMZGXnp8OHDnmPGjGm1atUqv2bNmhVdO5raQ2oAhKgJ+flw772wciXMnAnPPANKJtsUDUtFrtQdwdfXtygkJCQ/KSnJp0uXLhcBtNYqMTHx0LBhw87Yb7t+/fomZR2ntGmFAwMDi5KTk1NWr17tt2DBgsAVK1b4r1y5MqN4m8LCQqKjo6MA+vfvnzt37tyjFYk5ODi4ICMjw6P4dWZmpkd1Xv2D1AAI4XgXLhgd/VauhJdegmeflcJfiBrk7u6uP/744/3vvfde8wULFvgD9OvX7/Trr78eWDzt765duzzPnDnj0qtXr7xVq1b5FxYWcvToUbft27f79OzZ8xwYTQBpaWkeRUVFrFq1yr9nz555mZmZbkVFRYwdOzb3hRde+Hn37t1XNHG4ublRPNVvRQt/gMGDB+e+++67zW02G5s2bWrs6+tbVN0JgNQACOFIZ8/C4MGweTO8/rrR7i+EqHF+fn62Tz/9dF/v3r3DfX19i6ZNm5aTkZHh2aFDB6vWWvn7+xds2LBhf0JCQu62bdt8rFZre6WUnjVr1pHQ0NDCXbt2ER0dfW7y5MmhGRkZXj169DiTkJCQu337du8JEyZYbDabAnjuueeOVCau2bNnt5g/f/51J06ccO/YsWNUnz59Tq9YseJgfHz86fXr1zdp3bp1tLe3t+2NN97IqO7PxGHTAVcnmQ5Y1Em5ucakPt9+C0uXGj3+hahBMh1w9Vm3bp1vYmJiUFJS0j5nx1JZZU0HLDUAQjhCdrYxqc+ePUbV/913OzsiIYS4giQAQlS3o0ehb1/46SdYuxb693d2REKIXykuLi4vLi4uz9lxVCdJAISoThkZEBMDx4/DJ5/AHXc4OyIhhCiVJABCVJf0dOPK/+xZY1KfW25xdkRCCFEmSQCEqA67dkG/fsYwv5s3Q8eOzo5ICCGuScYBEOLX2r4devcGd3fYskUKfyFEnSAJgBC/xldfGW3+zZoZhX9ERPn7CCFqtRkzZlxXXceaPn16yDPPPBP0a47Rs2fPMF9f35v69OnTzn55Wlqax4033hgZGhoaPXDgwLYXL16s1AhjkgAIUVWffmr08G/Z0kgE2rRxdkRCiGowb9684Mpsb7PZKCpy3BQAjz322LGFCxf+VHL59OnTW06ZMiXr0KFDyU2aNCl85ZVXAipzXEkAhKiK1ath0CDjiv+rr+D6650dkRCiFOnp6R5t2rRpP3jw4DZt27Zt379//7Z5eXkua9eu9e3bt+8NxdutXr3ar1+/fjc8+OCD11+6dMklMjIyavDgwW0AZs6cGRQWFtY+LCys/XPPPdei+LgWiyV66NChlvDw8Pb79+/3WLVqlV9UVJQ1IiIiqnv37uHFx05NTfXu1q1bRMuWLTvMnj27UtMFA9x11115fn5+NvtlNpuNr7/+2nfcuHGnAMaPH3/i3//+d9PKHNdhnQCVUq2AZUAQoIFFWutXlFL+wArAAmQA8VrrU46KQ4hq9847MHYs3HwzfPwxNK3U35wQDVq3xd2uaie723r3yRm3z8jOu5TnErMsJqzk+ntvvDfn4VsePpGZl+l217/uusF+3fb7t6eXd86MjAyvhQsXZsTGxp4bPny45cUXXwycOXNm1tSpU0OPHj3qFhISUrhkyZLm48aNyxk5cuTppUuXtkhLS0sB2LJlS6Ply5c337lzZ6rWmi5dulhjYmLyAgICig4dOuT55ptv/hQTE5Nx9OhRtylTplg2b96cFhkZmZ+VlXV5WuB9+/Z5bdu2LT03N9fVarVGP/7449menp6/ahjerKwsN19f3yJ3d3cALBZLflZWlkc5u13BkTUAhcCjWuso4FbgIaVUFDAD2KS1DgM2ma+FqP2ys+Gpp2D0aOjVCz7/XAp/IeqA6667Lj82NvYcQEJCwolt27b5uLi4EB8ff2Lx4sX+OTk5rt99953P8OHDT5fcd/PmzT4DBgzI9fPzszVp0sQ2cODAU0lJSb4AwcHB+TExMefM7Rp369YtLzIyMh8gKCjocptAbGxsrre3tw4ODi709/cvOHLkSK24A89hQWitM4FM83meUioVuB64C+htbvY2sBl40lFxCPGrpaTA3LmwbBlcugQjRsCSJeDt7ezIhKhzrnXF7uvpa7vW+mDf4MKKXPGXVNo0vgAPPPDAiYEDB7bz8vLSgwYNOlV8NV1RjRo1spW/Fdhf7bu6ulJYWHhFQMuWLWv6/PPPhwAsWrQoo1evXufLO2ZQUFBhXl6ea0FBAe7u7mRkZHgEBQXlVyb+GukDoJSyAJ2Ab4EgMzkAOIbRRFDaPhOVUjuUUjuys7NrIkwhfqG1MZjPgAHQvj38859GtX9qKrz3nhT+QtQhmZmZHhs3bmwM8O677/r36NHjLIDFYikICgoqSExMDJ44ceLlCYvc3Nx08TTBffr0Obthw4ameXl5LmfOnHHZsGFDsz59+lw1JHDv3r3Pbd++3TctLc0DwL4JoDyjR4/OLZ4yuCKFP4CLiwu33npr3ltvvdUMYMmSJc3j4uJyK3pOqIEEQCnlA3wAPKK1PmO/ThtTEZbaDqK1XqS17qq17hoYGOjoMIUwXLpkzNzXsaMxsM9338Gf/wyHD8OCBRAZ6ewIhRCVZLFYLs6fP79F27Zt2+fm5ro99thjl68qR4wYcSI4ODi/c+fOF4uXjRo1KttqtUYNHjy4ze23335+5MiRJzp37mzt0qWLNSEhIfu22267UPIcISEhhfPmzcsYOnRou4iIiKihQ4e2ra74u3TpEpGQkND266+/9gsKCrrxgw8+8ANITEw8Mn/+/OtCQ0OjT5065TZ16tRKzbro0OmAlVLuwDrgU631HHNZOtBba52plAoGNmutr3nztEwHLBzuxAmjgP/HP+DYMYiOhunT4fe/By8vZ0cnRJXIdMBGb/24uLiwvXv37ilt/ejRo0M7dep0ftq0aXV+yuKy1Ph0wMpoZHkTSC0u/E1rgTHAX82faxwVgxDlSk832vfffhsuXDDu658+3RjTX1VqTA0hRB3Tvn17q7e3t23hwoWHnR2LMziyJ+JtQAKwWyn1g7nsjxgF//tKqQnAQSDegTEIcbXi8frnzIF168DTExIS4JFHjPZ+IUS9ERERkV/W1f+ePXtSazqe2sSRdwFsBcq6hIpx1HmFKFN+Prz/vlHwf/89BAbCs8/Cgw9Ci0qPzSGEEHVarbgXUQiHOnkSFi2C+fPh6FGwWmHxYhg1SnrzCyEaLEkARP21dy+88gq89RacP2/06n/zTYiNBRcZBVsI0bBJAiDqF62NWfnmzIG1a40pekeNMtr3b7zR2dEJIUStIZdBon4oKDAG6OnWDe64A7ZuhT/9CQ4eNEbtk8JfiAYpPT3dIywsrNb27h02bJjl+uuv7xAZGRkVGRkZtW3bNm8wJvsZO3Zsq9DQ0Ojw8PCorVu3Nqruc0sNgKjbcnON9vx58+DIEWN2vgULjF79jar970UIIard7NmzjxTP6lds5cqVTQ4cOOCVkZGRnJSU1PjBBx8M3bVrV1p1nldqAETddOAATJ0KLVvCE09AWJhxS19KCkyaJIW/EOIqKSkpHlarNerLL79sVFhYyKRJk1pGR0dbw8PDo1588cUAMK68J02a1DIsLKx9eHh41OLFi5sBrFu3zrdr164RvXv3bmexWKJHjhwZWlRURGFhIcOGDbMUbz9r1qxquaVozZo1TUeNGnXCxcWFmJiYc2fOnHE7ePBg5SYrKIfUAIi6Q2vYts1o3//oI6Mj3+9/D9OmQadOzo5OCFGe8eNbkZxcvdl5dPR5liwpdyCfH3/80XPEiBE3LFmy5Kfu3btfeOmllwKaNGlSlJycnHrhwgV18803Rw4aNOjMN99802j37t3eqampezIzM926detmjY2NPQuwe/fuxt9//31yeHh4fq9evcKWLVvWrF27dpcyMzPdi8cayMnJqfAcAMVmzZp1/QsvvBDcs2fPvH/84x9HvL29dWZmprvFYrk8uU9wcHD+wYMH3Vu3bl1Q2eOXRWoARO1XWGjcv9+9O9x+OyQlwZNPQkaGMUOfFP5CiGs4efKk25AhQ9q98847B7p3734BYOPGjX7vv/9+88jIyKhOnTpZT5065ZaSkuK1ZcsW3/j4+JNubm60atWq8JZbbjlb3P7eoUOHc1FRUflubm7Ex8ef3LJli09kZOSlw4cPe44ZM6bVqlWr/Jo1a1Z07WiuNGfOnJ8PHDiQ/OOPP6aeOnXK9emnn77OEZ9BaaQGQNRep08bt+298gocOgTt2sGrr8KYMdC4sbOjE8IhLhRcIP1EOqnZqaTmpPLEbU/g4+Hj7LCqRwWu1B3B19e3KCQkJD8pKcmnS5cuFwG01ioxMfHQsGHDrpikbv369U3KOk5p0woHBgYWJScnp6xevdpvwYIFgStWrPBfuXJlRvE2hYWFREdHRwH0798/d+7cuUftj1F8Re/t7a3Hjx9/IjExMQggODi4ICMjw6N4u8zMTI/qvPoHSQBEbZSRYXTqe+MNyMuDXr2M13Fx4Frp2jUhaqXTF0+TmpNKanYqKdkpxvOcVH469RPanCTVRbnwu6jfcWOQ3MXya7i7u+uPP/54f58+fcJ8fHxskydPPtmvX7/Tr7/+emBcXFyep6en3rVrl6fFYino1atX3uLFiwOnTJly4vjx427bt2/3mTdv3uFdu3Z57969u3FaWppHWFhY/qpVq/zvu+++7MzMTDdPT0/b2LFjc9u3b38xISHhilkA3dzcSEtLSykrtuJqfZvNxocfftjUarVeABg8eHDua6+91uL+++8/mZSU1NjX17dIEgBRf337LSQmwgcfGBPx3HOP0b7f1amTmQlRZVprjp87TmqOWcibV/WpOakczfvlQtDT1ZPw5uHcHHIzo28cjTXQijXASljzMLzcZDbK6uDn52f79NNP9/Xu3Tvc19e3aNq0aTkZGRmeHTp0sGqtlb+/f8GGDRv2JyQk5G7bts3HarW2V0rpWbNmHQkNDS3ctWsX0dHR5yZPnhyakZHh1aNHjzMJCQm527dv954wYYLFZrMpgOeee+5IZeK655572pw8edJNa62ioqLOL1u27CBAfHz86fXr1zdp3bp1tLe3t+2NN97IqO7PxKHTAVcXmQ64HisqMjr0zZljdPBr0sToxT9lCrRq5ezohKgQm7Zx+PThX67ks1NJyTEK/FMXf7m7y8fDh6jAKKwBRgFvDbQSFRhFm6ZtcHWp/totmQ64+qxbt843MTExKCkpaZ+zY6msGp8OWIhrysszBuh55RX46Sdo29ao5h87Fnx9nR2dEKUqKCpg/6n9l6/kiwv8tJw0zhecv7xdYKNArIFW4tvH/1LgB1q53vf6q9qRhXAWSQBEzTp0yJiUZ/Fio5PfbbfBSy/BXXdJ+76oNYo74pWstt97Yi8Ftl+aYVv5tcIaaGVi54mXq+2tgVYCGgU4MXrhCHFxcXlxcXF5zo6jOkkCIGrGjh1GNf/77xuvf/c7o33/llucG5do0HIv5v5SwNtd1WfkZlzREa+dfzusAVYGhw++XG0fGRBZf3rniwbJYQmAUmoJEAcc11pHm8v8gRWABcgA4rXWp8o6hqjjiorg3/82Cv4tW4yq/alT4eGHoXVrZ0cnGgitNVnnsq6qtk/NTiXzbObl7TxdPYkIiKDb9d0Y03GMUXUfaCXMPwxPN08nvgMhHMORNQBLgX8Ay+yWzQA2aa3/qpSaYb5+0oExCGc4d86YgnfuXNi/3yjs58yBCRPAz8/Z0Yl6yqZtHDp96Krb6kp2xPP18MUaaCX2htgr2ucd1RFPiNrKYQmA1vorpZSlxOK7gN7m87eBzUgCUH/8/DP84x+wcCGcOmVU77/wAgwdCm7S2iSqR0FRAftO7ruq2j79RHqpHfHuaX/P5fb5qMAoQnxDpCOeENR8H4AgrXVxndsxIKisDZVSE4GJAKGhoTUQmqiy776Dl1+Gf/0LbDa4+26YPt0YuleIKjpfcJ70nPSrbqvbe3IvhbbCy9uFNgnFGmDljtZ3XG6ftwZYad6ouROjF3XZjBkzrvvrX/96rDqONX369BAfH5+i5557Lquqx3B1de0SFhZ2ASAkJCT/iy++2AeQlpbmER8f3zY3N9etQ4cO5z/44IOfvLy8Knxvv9Muy7TWWilVZqBa60XAIjDGAaixwETF2Gywfr1Rtb95M/j4wEMPGW38bdo4OzpRhxR3xCtZbW/fEc9VuXKD/w1YA6wMiRxyudpeOuIJR5g3b15wZRIAm82G1hpXB93J5OnpaSttNMHp06e3nDJlStbEiRNPjRw5MvSVV14JePLJJ7MretyaTgCylFLBWutMpVQwcLyGzy9+rfPn4e23jfb9//3PmI73xRfhvvugaVNnRydqqeKOeCVvq0vJTuHY2V/+zxZ3xLul5S2MvWns5YJeOuKJqkpPT/fo379/WIcOHc4nJyc3Cg8Pv7By5cqMpKSkxvPmzWuxcePG/QCrV6/2e+211wLDwsIuXrp0ySUyMjIqPDz8wtq1a3+aOXNm0LvvvhsAkJCQkP3MM88cT09P9/jNb34T3qlTp7O7d+9uvGHDhr27du3yeuaZZ64vKipS/v7+hV9//fX/AFJTU727desWcfToUY/JkydnPfXUU7+67LPZbHz99de+a9asOQAwfvz4EzNnzgypzQnAWmAM8Ffz55oaPr+oqsxMYyKe11+HkyeN4Xnfew+GDQP3ap2iWtRhNm3jYO7BUse4z72Ye3k7Xw9fogKj6N+u/+W2eWuAFUtTi3TEq++6dYu4atndd59kxoxs8vJciIkJu2r9vffm8PDDJ8jMdOOuu264Yt327enlnTIjI8Nr4cKFGbGxseeGDx9uefHFFwNnzpyZNXXq1NCjR4+6hYSEFC5ZsqT5uHHjckaOHHl66dKlLYqvuLds2dJo+fLlzXfu3JmqtaZLly7WmJiYvICAgKJDhw55vvnmmz/FxMRkHD161G3KlCmWzZs3p0VGRuZnZWVd/iLv27fPa9u2bem5ubmuVqs1+vHHH8/29PSscM12fn6+S3R0tNXV1VU/9thjxxISEnKzsrLcfH19i9zN/78WiyU/KyvLo5xDXcGRtwG+h9HhL0ApdQR4FqPgf18pNQE4CMQ76vyimvz4o9G+v3y5MS3vkCFG+/5ttxnj9YsGyb4jnv1tdWk5aVwovHB5uxaNW2ANsDKi/YjLt9VZA6zSEU/UqOuuuy4/Njb2HEBCQsKJefPmtXBxccmKj48/sXjxYv+HHnroxHfffefz4Ycf/lRy382bN/sMGDAg18/PzwYwcODAU0lJSb7Dhw/PDQ4Ozo+JiTlnbte4W7dueZGRkfkAQUFBl6cFjo2NzfX29tbe3t6F/v7+BUeOHHG74YYbKjyxz969e3e1adOmICUlxaNfv34RnTt3vuDv71+paYdL48i7AH5fxqoYR51TVBObDT75xGjf37QJGjUyxuefOtWYklc0GOcLzpOWk3ZVtf2+k/tK7YjX29L7crW9dMQTpbrWFbuvr+2a64ODCytyxV9SadP4AjzwwAMnBg4c2M7Ly0sPGjTolHslazMbNWpkq8h29lf7rq6uFBYWXhHQsmXLmj7//PMhAIsWLcro1avXefv1bdq0KQCIiorKv/XWW/O2b9/eaMyYMafy8vJcCwoKcHd3JyMjwyMoKCi/MvHLvVniFxcuwDvvGFf8qakQEgJ//StMnAjNmjk7OuFAcfAFIAAAExFJREFUpy6cuuq2utScVA7mHryqI15UYBRDI4derrqPCIiQjniiVsvMzPTYuHFj4759+5579913/Xv06HEWwGKxFAQFBRUkJiYGf/LJJ/8r3t7NzU1funRJeXp66j59+pwdP3685c9//vMxrTUbNmxotnTp0gMlz9G7d+9z06dPb52WluZR3ARgXwtwLaNHj84dPXp0bmnrsrOzXX18fGze3t46MzPTbceOHT5//OMfj7m4uHDrrbfmvfXWW80mTpx4asmSJc3j4uJKPUZZJAEQkJUFr71mPHJyoFMnIxEYPhw8KtWkJGoxrTXHzh4rdWpa+454Xm5eRDSP4NaWtzLupnGX2+fb+beTjniiTrJYLBfnz5/fYuLEiY3CwsIuPvbYY5c7yo0YMeLEq6++6ta5c+eLxctGjRqVbbVao6Kjo8+vXbv2p5EjR57o3LmzFYxOgLfddtuF9PT0K/45hoSEFM6bNy9j6NCh7Ww2G82bNy/Ytm3b3l8b+w8//OD10EMPtVZKobXmkUceOdalS5eLAImJiUfuueeeG2bPnn19+/btz0+dOrVSsy7KdMANWXKycbX/zjuQnw+DBhnt+3fcIe37dVhxR7zSpqY9fen05e38PP2u6IBXXG0vHfHqD5kO2LgLIC4uLmzv3r17Sls/evTo0E6dOp2fNm1anZ+yuCwyHbAwaA2ffWa073/2GXh7G0P0PvIIhIc7OzpRCflF+UZHvBLV9uk56Vd1xIsKjGJkh5FXzEEf7BMsHfFEg9a+fXurt7e3beHChYedHYszSALQUFy8aPTknzMH9uyB666Dv/zF6NzXXDpq1Wbn8s+RfiL9qtvqSnbEa92kNdZAK30sfa64qvf39ndi9EI4V0RERH5ZV/979uxJrel4ahNJAOq77Gzj3v1XX4Xjx+HGG2HpUhgxAjylPbc2OXnhZKlT0x48ffDyNq7K1ZiaNtDK3ZF3X662l454ohaz2Ww25eLiUvvbm+shm82mgFLvVpAEoL5KTTXa95ctg0uXYMAAo33/zjulfd+JtNZkns28oqAvbp/POvfLUOHFHfF6tOrBhE4TLlfbt/Nvh4erdMwUdUpydnZ2VGBg4GlJAmqWzWZT2dnZTYDk0tZLAlCfaA1ffAGJifDxx+DlBWPGGO37Vquzo2tQbNpGRm5GqWPcl+yIFxUYxYCwAb90yAu00rpJa+mIJ+qFwsLC+44dO/bGsWPHouH/27v34Liq+4Dj359WK2lXEtZjZVl+YRscG5sYsD3BJC0EAzaYDLQh6RBoQ0iYhKY0TUMJkMdMwqSBph2mZEiaEnBaOoQQKE0hA2VcHkOHwWBs/FCwARdS4zcSxpa0K+1q9esf56x0Je1aD8u70u7vM3Nn7z337H3sWel37rln76Gs0MdTYvqA1t7e3huyrbQKQDHo6XEj8d19N2zfDtOnwx13wI03QlNToY+uqGU64g39Wd2utl109/b/qojm6mbOaDqDaz56zaD789YRr7iouuEyOjqgs9NNwfmhy6PJt23b1H7+1ooVKw4DVxT6OMxwVgGYyg4cgPXr4d574eBBWLoUHngArrnGXf2bE5JMJ2mPt9MWb6M94V/j7ew5umfQE/HSOvCsj0xHvNXzVvffn7eOeJOTqusbO9pAPJp8XV1uu6MRCkFtrZtqatxUW+v65GbmM+nGnAxWATgBPd0pNj79Ihf88Ul+urEq7N0LW7YMnvbvd+vXrnUj9F1yid3fz6G7t5v2ePugQN4Wbxsc3Ies60h2ZN1WSEIsbFzIkqYlXHXGVf335xc1LqK6ojrPZ1Y6enrGduU8Ur7OTkiP8mnqZWXDg3JNDcyaNTh4Z5vPta6iwv5cTWFZBWCc/mv9oyz9+tUsTivxQ11EayITs+G+PnjnneHBvr3drS8rg8WLXWe+5cthzRp35V9CEqlEf7AOBuyhgTy4rivVlXN7tRW1xKIxGqONxKIxFscW0xhx8/2v0YHlpuom64g3glRqdEF5LAE7NeqhU7IH3unT4bTTxhewq6osWJviYxWAUdrx8g42/uC7lDU286UH/5mPXX4RG+9s4cjFn+Wq8f5j6O2FN990Af711wdejx1z68NhOPNMNwLf8uVuWrbMDc5TJOKpeO5AHm+nLTF8XTwVz7m9aZXT+oP1jJoZLG1amjWQZ9Iao40lH8zT6eFXxmO9Tz10vqdn9PuPRocH3vp6mDt35KvobMvRqKsnG2OOzyoAx/H+/jae/M73mPPcI1y4p42PKjyxaC4ADc0NrHt77+g3lky6B/AEr+q3bXMD8IB7It9ZZ8G11w4E+6VLp8xv9VWVrlRX1kAeDOZD1wU7yg1VV1XXH6xn1s5kWfOynIE8Fo3REGkgHBrbaF5TTV/f8E5mJxqwE4mR95tRVTU88NbWQkvL6Jq9h85XV7t74caY/LMKwBB96T7KQu7y4dULzuKLu/ezpzbEwx8/j3lf+xZX/MmnRt5IPO564weDfWvrQBtmba0bcOcrXxkI9osWQfnkKA5VpSPZkTuQ52huT6azj0QpCPWR+v5gPXfaXM5pOYdYJDaseT2z3BBpoLxscnwe46XqgutE9AQPdjIbrXA4e1BuahrfPevqardNY0xxmNr/YSfQcw89wXs/vZMLWjdx5D9e4pzV55L+i+/zeGcHV9z6l/xZOMtHperuzWeu7DPN+Dt3uks1cF16ly93D+HJBPsFC/LWRqmqHOs5NrzDW7bgHliX6st+w7VMymiINPQH63l181jZsjJnII9FY9RX1RfsN+3ptGt8Od6USo2cJzhlgvpIAbuzc+BrMJKysuE9wmtqYM6cE+tkZowxuRSkAiAilwL3ACHgflW9K5/7z1zlt768lW1fu54z9rzF6sNxegVeOLWJ0MFD0NnJFevOh3374NePuB73+/a51+B8MnDVO3OmC/Cf/vRAsJ8zZ1y9h/q0j+7ebhKpBIneBIlUwi37+USvWz7affS4gbw90T7oefFBIQnREGnob0o/veF0Vs1alTOQx6Ix6qrq0L6y4wfROCQ/hMNJ2DuGwDpRATo4jTYAj4VI9sA7Y4b7vfZYmsAzy5WV1snMGJNfeR8OWERCwFvAJcBeYBPwOVV9I9d7xjsccLI7yYaf/5Ij//Ms0be3M+vQHuYdPcb2hcu45Ju3cGDHThp/9APaImHi9TNoaW4h2nUU9h9AMh3xAvqiEVIt00lOj9HT3EiiuZHuWB1HT53BwY/M4oNpVXQlEySS3cRTCeLJBPGUD9ipwcG7p7ebRDpBd6+betLd9KQTdKcT9KQTpDR7c3ouIcqpLmskSowojUQ0RmVfI5XpGBW9jVT0xihPxQj1NBLqiSGJRrR7Gqnk8GBeiKAK7g5IRcXET+HwxG/PgrWZCibDcMBm8ipEBeA84HuqutYv3w6gqnfmes94KwDPnzaXC98ZeZTHZBnsrx2Y9p0SmA+kd4zn2TrpMKQi0BuB3qqB+ZRf7p8fsv64eQPzPadAPOZeGR6VJjqoTnQwtaBqzMljFQBzPIW4BTALCEblvcC5QzOJyJeBLwPMnTt3XDvacta5HAr3cKimlv3TmuiMROisrKSzMkI8XEVXZYTOygjJ8ipCUkFIwoQID7xSQbmEmSNh5qXDhOJhyiVMSFy6mw9TUVZJRShCpUSoCFVRFYpQFYpQUVZFuDxEKOR6OpeV0T+fK22k5aFp2QJyJi0ctp9DGWOMyW7SdgJU1fuA+8C1AIxnGzc//uiEHpMxxhhTLApxfbgPmBNYnu3TjDHGGJMnhagAbAIWish8EakArgaeKMBxGGOMMSUr77cAVLVXRG4CnsH9DHC9qv4u38dhjDHGlLKC9AFQ1aeApwqxb2OMMcYU5haAMcYYYwrMKgDGGGNMCbIKgDHGGFOCrAJgjDHGlKC8Pwp4PETkfeD/RsgWA9rycDiTjZ13abHzLi0net6nqmrTRB2MKS5TogIwGiLyWik+89rOu7TYeZeWUj1vkx92C8AYY4wpQVYBMMYYY0pQMVUA7iv0ARSInXdpsfMuLaV63iYPiqYPgDHGGGNGr5haAIwxxhgzSlYBMMYYY0rQlK8AiMilIvKmiOwWkdsKfTzjISJzROR5EXlDRH4nIn/l0xtEZIOIvO1f6326iMiP/TlvF5HlgW1d5/O/LSLXBdJXiMgO/54fi4jk/0yzE5GQiLwuIr/1y/NF5BV/rI/4YaMRkUq/vNuvnxfYxu0+/U0RWRtIn5TfDxGpE5HHRGSXiOwUkfNKobxF5K/9d7xVRB4WkapiLW8RWS8ih0WkNZB20ss41z6MGUZVp+yEG074f4EFQAWwDVhS6OMax3m0AMv9fC3wFrAE+BFwm0+/Dfg7P78OeBoQYBXwik9vAN7xr/V+vt6ve9XnFf/eywp93oHz/wbwS+C3fvnXwNV+/mfAn/v5rwI/8/NXA4/4+SW+7CuB+f47EZrM3w/gX4Eb/HwFUFfs5Q3MAt4FIoFy/kKxljdwPrAcaA2knfQyzrUPm2waOk31FoCPAbtV9R1VTQK/Aq4s8DGNmaoeUNUtfr4D2In7Z3klLlDgX//Iz18JPKjORqBORFqAtcAGVf1AVY8AG4BL/bpTVHWjqirwYGBbBSUis4HLgfv9sgCrgcd8lqHnnfk8HgMu8vmvBH6lqj2q+i6wG/fdmJTfDxGZhgsODwCoalJVP6QEyhs3BHlERMqBKHCAIi1vVX0R+GBIcj7KONc+jBlkqlcAZgHvBZb3+rQpyzdzngO8AjSr6gG/6iDQ7Odznffx0vdmSZ8M/hH4JtDnlxuBD1W11y8Hj7X//Pz6oz7/WD+PQpsPvA/8wt/6uF9Eqiny8lbVfcA/AHtwgf8osJniL++gfJRxrn0YM8hUrwAUFRGpAf4d+LqqHguu87X8ovrNpoh8CjisqpsLfSx5Vo5rGv4nVT0H6MI11fYr0vKux12dzgdmAtXApQU9qALKRxkX4/fITJypXgHYB8wJLM/2aVOOiIRxwf8hVX3cJx/yTX3418M+Pdd5Hy99dpb0QvsEcIWI/B7XXLsauAfX/Fnu8wSPtf/8/PppQDtj/zwKbS+wV1Vf8cuP4SoExV7eFwPvqur7qpoCHsd9B4q9vIPyUca59mHMIFO9ArAJWOh7EVfgOgo9UeBjGjN/X/MBYKeq3h1Y9QSQ6fV7HfCfgfTP+57Dq4CjvsnvGWCNiNT7q601wDN+3TERWeX39fnAtgpGVW9X1dmqOg9Xds+p6rXA88BnfLah5535PD7j86tPv9r3Gp8PLMR1kJqU3w9VPQi8JyKLfNJFwBsUeXnjmv5XiUjUH1fmvIu6vIfIRxnn2ocxgxW6F+KJTrjes2/hev9+u9DHM85z+ANcM912YKuf1uHudz4LvA38N9Dg8wvwE3/OO4CVgW19EdcpajdwfSB9JdDq33Mv/imQk2UCPsnArwAW4P6h7wYeBSp9epVf3u3XLwi8/9v+3N4k0ON9sn4/gLOB13yZ/wbXw7voyxv4PrDLH9u/4XryF2V5Aw/j+jqkcK0+X8pHGefah002DZ3sUcDGGGNMCZrqtwCMMcYYMw5WATDGGGNKkFUAjDHGmBJkFQBjjDGmBFkFwBhjjClBVgEwRU1E0iKyVUS2icgWEfn4CPnrROSro9juCyKycpzH9JSI1I3nvcYYM1GsAmCKXUJVz1bVs4DbgTtHyF+HG4XupFHVdeoG/zHGmIKxCoApJacAR8CNuyAiz/pWgR0ikhk17i7gNN9q8Pc+760+zzYRuSuwvc+KyKsi8paI/OHQnYlIi4i86LfVmskjIr8XkZiI3OjXbRWRd0Xkeb9+jYi87I/tUT9GhDHGTCh7EJApaiKSxj1ZrQpoAVar6ubMcLSqekxEYsBG3CNlT8U9kfBM//7LgO8CF6tqXEQaVPUDEXkB2KyqN4vIOuAbqnrxkH3fDFSp6t+KSMjvr8OPfbBSVdt8vjDwHG4c95dxz8i/TFW7RORW3JPx7jiZn5MxpvSUj5zFmCktoapnA4jIecCDInIm7tGrPxSR83FDEc8i+7CpFwO/UNU4gKoGx3fPDNq0GZiX5b2bgPU+wP9GVbfmOMZ7cM+5f9KPkLgEeMk94p0KXKXAGGMmlFUATMlQ1Zf91X4T7pnxTcAKVU35q/KqMW6yx7+myfK3pKov+grG5cC/iMjdqvpgMI+IfAHX6nBTJgnYoKqfG+OxGGPMmFgfAFMyRGQxEMINKTsNOOyD/4W4IAzQAdQG3rYBuF5Eon4bDWPY36nAIVX9OXA/bsjf4PoVwN8Af6qqfT55I/AJETnd56kWkY+M7UyNMWZk1gJgil1ERDJN7wJcp6ppEXkIeFJEduBG5dsFoKrtIvKSiLQCT6vqLSJyNvCaiCSBp4BvjXLfnwRuEZEU0IkbsjXoJqABeN4397+mqjf4VoGHRaTS5/sOboQ7Y4yZMNYJ0BhjjClBdgvAGGOMKUFWATDGGGNKkFUAjDHGmBJkFQBjjDGmBFkFwBhjjClBVgEwxhhjSpBVAIwxxpgS9P/wYDLfMIL/6wAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAEWCAYAAAAQHy/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdeVzU1f748ddhRwEVQQQVxwVkECuVLCtNQ80SNTPJNFzLbLmZZuXttmi35d6faaaVuVaWdVXKNLUsC82+1PVqiyJLLuGSiICiuLHN+f3x+WCIIKAMg/J+Ph7zgPms7xmGOe/POedzjtJaI4QQQoi6xcnRAQghhBCi5kkCIIQQQtRBkgAIIYQQdZAkAEIIIUQdJAmAEEIIUQdJAiCEEELUQZIAXCGUUlop1fYi699VSj1f4vnDSqkMpdRJpVTjSzifxTyny6XGXF3M19C6Bs4TbJ7LuQbOFaCU+l4plauUmmHv84nKU0o9q5Ra6Og4hLA3SQBKUUqlKaXylVJ+pZb/YhaIFvP5++bzgaW2e8NcPsp8PkopVWQWLCeVUn8opd5TSoVWZ9xa6/Fa63+a53QFZgJ9tNZeWuvsihKIy2G+xh+q6VgblVIPlFxmvoa91XH8UudKU0r1KnGe/ea5iqr7XGUYB2QBPlrrJy/3YEqpQKXUaqXUoZKf0xLr3ZVSi5VSJ5RSh5VSk0qtj1JKpSilTiul4pVSLS83JkdQSk1VSn1Uhe17KKUOllymtX5Va/1AefsIcbWQBKBsfwD3FT9RSnUA6pWx3e/AiBLbuQAxwJ5S2/2otfYCGgC9gDPANqVURHUEW8YVawDgAeysjuMLu2gJJOlLGImrnFoZG/AVMLic3aYCIeZ5ewJPK6X6msfzAz4Dngd8ga3AsqrGVVW1oXZJiDpNay2PEg8gDXgO+F+JZa8D/wA0YDGXvW8uzwAamcuigS+BH4BR5rJRwA9lnGcNEHeROJ4C0oFDwBjz3G1LnHsusA44hZFUvA+8DISayzRwEvgO+N58fspcdm8Z53M2X08WsBd41NzHxVzfAFhkxvSneS5nwAqcBYrMY+eY27ubx9tvvkfvAp4lzjcQ+BU4gZEw9QVeMY9z1jzWW+a2JV97A2AJkAnsM/9WTiXfa/O8xzASuTvKeX8/xCg0z5jnehqwlHrNG83XmWBu8wXQGFhqxv2/4s+DuX0Y8A1wFEgFYso59/tAAZBvHreX+X7NMv/eh8zf3c3tewAHgWeAw8CHF/ncuFDic1pi+SGMGqHi5/8E/mP+Pg5IKLGuvvm+hF3kf+TvQJL5Pr8HeJRYH23+bXPM9+6aUvs+A2wH8sx40zA+79sxPqOLMJLYL4FcYAN//Y/1AA6WEU8vjM9QvvnengR+M9ePBpLNY+0FHir1Om3m9ieBIIxk6aMSxx+AkUznmJ8Ja6lzTzZjP46ROHmY6/ww/s9zzM/EZszPqjzkURseDg+gtj1KfJmkYhRuzuaXb0suTABeBuYDD5vLlmPUHFQmARgDZJQTQ1+MQjPC/JL6mAsTgOPAzRi1OB7F8ZjrLZQoyMxl5/Yv55zjgRSgBcZVYDznF4YrgXlmPE2ALSW+SC94jcAbwGrzWN4Yhedr5rouZvy9zfibYRY25hfsA6WOVfK1LwFWmce0YNTCjC0RRwHwoPl3exij4FMX+1uXeH7e+2bGshtog5F4JJnn64VRcC0B3jO3rQ8cwChsXICOGMlUeDnnPvf3Mp+/BPxkvrf+GAXnP811PYBC4N8YiYJnWcc0t70gAQAamcsCSiy7B9hh/v4mMLfUcRKBwRd53xJLfFb+j78+ex2BI8AN5t9gpLm9e4l9fzX39Syx7CeMQr+Zuf/P5rE8MJLYF0u8F2UmAObvUylReJvL+pl/QwXcCpwGOl3keOeOwV8JdW/AFSNR3A24lTj3FozEwRcj0RhvrnsNI/F1NR/dKOezKA95OOIhTQDl+xCjer83xj/1n+VstwQYoZRqiPHl8nklj38I4wujLDEYBUui1voUxhdSaau01v+ntbZprc9W8pwXEwPM0lof0FofxfjyAowOa8CdwBNa61Na6yMYBfzQsg6klFIYV5UTtdZHtda5wKslth8LLNZaf2PG/6fWOqWiAM2mjqHA37XWuVrrNGAGEFtis31a6wXaaMf/AAjEKFgu1Xta6z1a6+MYV6R7tNYbtNaFwAqMQgqMq940rfV7WutCrfUvwKfAkEqeZzjwktb6iNY6E5hW6nXZMArBPK31mSq+Bi/z5/ESy45jJFHF649zvpLry/JWic/KK/zVZDYOmKe1/q/Wukhr/QHGlf6NJfadbe5b8nXM0VpnaK3/xLhS/q/W+hfzs72Sv97nKtNarzX/hlprvQn4GqMwrox7gbXmZ7UAo3bJE7ip1Os5ZL4XXwDXmcsLMD5/LbXWBVrrzVprmXxF1BrSBle+DzGqzlthFPJl0lr/oJTyx2giWKO1PmOUfxVqhlEtWJYgYFuJ5/vK2OZAZU5SBUGljlnynC0xrmDSS7w2p4vE4I/RZ2Jbie0VxhUhGFd/6y4hRj8zjpKx7cN4L4sdLv5Fa33aPL8Xly6jxO9nynhefOyWwA1KqZwS610wPkeVEcSFryuoxPPMy0j0Tpo/fTCaV4p/zy2x3qfUPiXXl6X0Z6U41pbASKXU30qsd+P811LW56ay73OVKaXuAF7EuJp3wvhs7qjk7uf9XbTWNqXUAcr5zGHULhS/1ukYyfvX5udwvtb6X5fwEoSwC6kBKIfWeh9GG/KdGB2kLuYj4EkukiiUYRDGlU5Z0jEKyWLBZYVYhXNVxsXOeQDjKs5Pa93QfPhorduXE0sWxpd2+xLbN9BGR8ji47UpJ46Lva4sjKuqkj3Ugym/dqYi1fkeHgA2lXi9DbVxR8HDldz/EBe+rkMlnl9yrFrrYxh/32tLLL6WvzqJ7iy5TilVH+Pvc7FOpKU/K8WxHgBeKfU+1NNaf1IypEt7JYBRHX+uQ65ZK+Rf3rGVUu4YNTGvYzSBNMRIPlVZ25fhvL+LWbvVgkp85sxaqie11q0x+hFMUkpFVbSfEDVFEoCLGwvcZlbDX8xsjKaC7y+2kVLKWSnVSik1B6PtcVo5my4HRimlwpVS9TCuXi5XBnCxe+mXA48rpZorpRoBU4pXaK3TMapNZyilfJRSTkqpNkqpW0scu7lSys3c3gYsAN5QSjUBUEo1U0rdbm6/CBht3nrmZK4LqyhOs1p/OfCKUsrbvFVtEkYCdikqek+qYg0QqpSKVUq5mo/rlVLWSu7/CfCcUsrf7JX/AlV8XUopD4w+AgDu5vNiS8zjNzLf6wcx+iGAUcUeoZQabO7zArC9gmaZR83Pii9G7VfxXQMLgPFKqRuUob5Sqp9S6mLNCVXxO+BhHtMVoxOoe4n1GYBFKVX83eZmrs8ECs3agD6ltm+slGpQzvmWA/3Mz6orRqKfh9FH46KUUtFKqbZm0nAco4OrrbIvVAh7kwTgIsx2w62V2O6o1vrbi7TvdVVKncToOb4Ro3r1eq11mdWQWusvMXqBf4fR4ei7S4m/lKnAB0qpHKVUTBnrFwDrgd8wOmCVrvUYgfFlWtzzOw6jfRMzvp3AYaVUlrnsGTP2n5RSJzB6crczX98WjM5yb2B8MW7ir6usN4F7lFLHlFKzy4jzbxhXgXsxOlt+DCyu3FtwgdcwCsUcpdTkSzwGYFztYRQsQzGuGg/zV6e9yngZ4/a77RjV0z+by6qi+I4GMDp0lmxjfxHjbot9GO/3dK31V2bsmRi3D76C8be9gXL6d5TwMUZSuNc87svmsbZiJBdvmcfajdE5s1qYfTEeARZiXIWfwuikW2yF+TNbKfWz+Xd5HKMgPwYMw+icWny8FIzka6/5OSjZVIHWOhW4H5iDUQPVH+ivtc6vRLghGJ/7k8CPwDta6/iqvWIh7EdJnxQhRFUopdIw7tTY4OhYhBCXTmoAhBBCiDpIEgAhhBCiDpImACGEEKIOkhoAIYQQog66IgYC8vPz0xaLxdFhCCHEFWXbtm1ZWmv/ircUddEVkQBYLBa2bq3wbjwhhBAlKKXKGkVUCECaAIQQQog6SRIAIYQQog6SBEAIIYSogyQBEEIIIeogSQCEEEKIOkgSACGEEKIOkgRACCGEqIPsmgAopSYqpXYqpRKVUp8opTyUUq2UUv9VSu1WSi0rnkNeCCFECRkZ8MQTkJfn6EjEVcpuCYBSqhnGPNyRWusIwBljjvF/A29ordtizM891l4xCCHEFenAAejeHRYsgJ07HR2NuErZuwnABfBUSrkA9YB04DYgzlz/AXCXnWMQQogrx5490K0bHD4M69dDp06OjkhcpeyWAGit/wReB/ZjFPzHgW1Ajta60NzsINCsrP2VUuOUUluVUlszMzPtFaYQQtQeSUlG4X/yJHz3Hdxyi6MjElcxezYBNAIGAq2AIKA+0Ley+2ut52utI7XWkf7+MpeFEOIq9/PPRrW/1rBpE3Tu7OiIxFXOnk0AvYA/tNaZWusC4DPgZqCh2SQA0Bz4044xCCFE7ZeQAD17Qv36sHkztG/v6IhEHWDPBGA/cKNSqp5SSgFRQBIQD9xjbjMSWGXHGIQQonbbsAF694aAAPjhB2jb1tERiTrCnn0A/ovR2e9nYId5rvnAM8AkpdRuoDGwyF4xCCFErfbFF9CvH7RuDd9/Dy1aODoiUYe4VLzJpdNavwi8WGrxXqCLPc8rhBC13rJlcP/90LEjfPUV+Po6OiJRx8hIgEIIUdMWLYL77oObbjKaAKTwFw4gCYAQQtSkN9+EBx6APn3gyy/Bx8fREYk6ShIAIYSoKa++agzvO2gQrFoF9eo5OiJRh0kCIIQQ9qY1/P3v8I9/GO3+y5eDu7ujoxJ1nF07AQohRJ1ns8Hjj8Pbb8P48cZPJ7n2Eo4nn0IhhLCXwkIYO9Yo9CdPhnfekcJf1BpSAyCEEPaQnw/Dh0NcHEybBs8/D0o5OiohzpEEQAghqtuZM3DPPbBuHcyYAZMmOToiIS4gCYAQQlSn3FwYOBA2boR582DcOEdHJESZJAEQQojqcuwY3HEHbN0KH35oNAEIUUtJAiCEENXhyBFjcJ/kZKPd/667HB2REBclCYAQQlyugweNGf327TMm+OnTx9ERCVEhSQCEEOJy7N0LUVGQnQ3r10O3bo6OSIhKkQRACCEuVXIy9OoFZ8/Cd99BZKSjIxKi0mRECiGEuBS//ALdu0NREWzaJIW/uOJIAiCEEFX144/Qsyd4esLmzRAR4eiIhKgyuyUASql2SqlfSzxOKKWeUEr5KqW+UUrtMn82slcMQghR7b77zujw5+8PP/wAISGOjkiIS2K3BEBrnaq1vk5rfR3QGTgNrASmAN9qrUOAb83nQghR+61ZA3feCa1awfffQ3CwoyMS4pLVVBNAFLBHa70PGAh8YC7/AJCbZYUQtd/y5TBoEHToYIzyFxjo6IiEuCw1lQAMBT4xfw/QWqebvx8GAsraQSk1Tim1VSm1NTMzsyZiFEKIsr33Htx3H9x4I3z7LTRu7OiIhLhsdk8AlFJuwABgRel1WmsN6LL201rP11pHaq0j/f397RylEEKUY84cGDPGuN3vq6/Ax8fREQlRLWqiBuAO4GetdYb5PEMpFQhg/jxSAzEIIUTVvfYaPP64Mazv6tVQv76jIxKi2tREAnAff1X/A6wGRpq/jwRW1UAMQghReVrDs88aj2HDjPZ/d3dHRyVEtbJrAqCUqg/0Bj4rsfhfQG+l1C6gl/lcCCFqB5sNJkwwrv7HjYMlS8DV1dFRCVHt7DoUsNb6FNC41LJsjLsChBCidikqggcfNDr9TZoEr78OSjk6KiHsQkYCFEIIgPx8o7r/vffgxRel8BdXPZkMSAghzp6FIUOMgX6mT4fJkx0dkRB2JwmAEKJuO3kSBg6E+HiYOxfGj3d0RELUCEkAhBB1V06OMbTvli1GZ7/773d0RELUGEkAhBB1U2Ym9OkDO3fCihXGML9C1CGSAAgh6p4//zRG9ktLMwb46dvX0REJUeMkARBC1C1//AFRUZCVBevXQ/fujo5ICIeQBEAIUXekpBhX/qdPG5P6XH+9oyMSwmEkARBC1A2//mq0+StlTOd7zTWOjkgIh5KBgIQQV7+ffoKePcHDAzZvlsJfCCQBEEJc7eLjjWr/xo2Nwj801NERCVErSAIghLh6rVtn3OdvsRiFf8uWjo5IiFpDEgAhxNUpLg7uugvCw402/8BAR0ckRK0iCYAQ4urzwQdw773QpQt89x34+Tk6IiFqHUkAhBBXl7ffhlGj4LbbjPv8GzRwdERC1Ep2TQCUUg2VUnFKqRSlVLJSqqtSylcp9Y1Sapf5s5E9YxBC1CH//jc89hgMGABffAH16zs6IiFqLXvXALwJfKW1DgOuBZKBKcC3WusQ4FvzuRBCXDqt4bnnYMoUuO8+o/3fw8PRUQlRq9ktAVBKNQC6A4sAtNb5WuscYCDwgbnZB8Bd9opBCFEHaA0TJ8Irr8ADD8CHH4Krq6OjEqLWs2cNQCsgE3hPKfWLUmqhUqo+EKC1Tje3OQwElLWzUmqcUmqrUmprZmamHcMUQlyxiorgwQfhzTfhiSdg/nxwdnZ0VEJcEeyZALgAnYC5WuuOwClKVfdrrTWgy9pZaz1fax2ptY709/e3Y5hCiCtSQQEMHw6LFsHzz8PMmcYwv0KISrFnAnAQOKi1/q/5PA4jIchQSgUCmD+P2DEGIcTV6OxZGDwYli0zOv699JIU/kJUkd0SAK31YeCAUqqduSgKSAJWAyPNZSOBVfaKQQhxFTp5EqKjjV7+b78NTz/t6IiEuCLZezbAvwFLlVJuwF5gNEbSsVwpNRbYB8TYOQYhxNUiJwf69TMm9/ngAxgxwtERCXHFsmsCoLX+FYgsY1WUPc8rhLgKZWbC7bdDYiIsX240AQghLpm9awCEEOLyHTpkzOj3xx+wahXccYejIxLiiicJgBCidktLg6goOHIEvvwSevRwdERCXBUkARBC1F6pqcaV/8mTsGED3HCDoyMS4qohCYAQonbavh169zZG+tu4Ea691tERCXFVkdkAhRC1z5YtRlW/mxts3iyFvxB2IAmAEKJ22bTJaPNv1Mgo/Nu1q3gfIUSVSQIghKg9vvwS+vaF4GCj8LdYHB2REFctSQCEELXDp5/CwIFgtRq1AEFBjo5IiKuaJABCCMdbsgRiYuD66+G778DPz9ERCXHVkwRACOFYc+fCyJHQsyesXw8NGzo6IiHqBEkAhBCOM306PPII9O8Pa9aAl5ejIxKizpAEQAhR87SGF14wZvK7916j/d/Dw9FRCVGnyEBAQoiapTU8+SS88QaMGQPz54Ozs6OjEqLOkRoAIUTNKSqCceOMwv/xx2HBAin8hXAQSQCEEDWjoABiY2HhQvjHP2DWLHCSryAhHEWaAIQQ9nf2LAwdakzl+69/wTPPODoiIeo8uyYASqk0IBcoAgq11pFKKV9gGWAB0oAYrfUxe8YhhHCgU6dg0CD45ht46y149FFHRySEoGaaAHpqra/TWkeaz6cA32qtQ4BvzedCiKvR8eNw++3w7bfw3ntS+AtRiziiAW4g8IH5+wfAXQ6IQQhhb1lZxqQ+//0v/Oc/MGqUoyMSQpRg7wRAA18rpbYppcaZywK01unm74eBgLJ2VEqNU0ptVUptzczMtHOYQohqlZ4Ot94KiYnw+ecwZIijIxJClGLvToC3aK3/VEo1Ab5RSqWUXKm11kopXdaOWuv5wHyAyMjIMrcRQtRC+/YZV/6HDxuz+/Xs6eiIhBBlsGsNgNb6T/PnEWAl0AXIUEoFApg/j9gzBiFEDfr9d+jWDbKzYcMGKfyFqMXslgAopeorpbyLfwf6AInAamCkudlIYJW9YhBC1KAdO6B7d+OWv/h4uPFGR0ckhLgIezYBBAArlVLF5/lYa/2VUup/wHKl1FhgHxBjxxiEEDXhf/8zevvXq2dc+YeFOToiIUQFKkwAlFJdgfuBbkAgcAbjSn4t8JHW+nhZ+2mt9wLXlrE8G4i6jJiFELXJ999DdDT4+Rm3+7Vq5eiIhBCVcNEmAKXUl8ADwHqgL0YCEA48B3gAq5RSA+wdpBCillq/Hvr2hWbNYPNmKfyFuIJUVAMQq7XOKrXsJPCz+ZihlPKzS2RCiNrts8+M4X3bt4evvwZ/f0dHJISogovWABQX/maHPifz91Cl1ACllGvJbYQQdchHH0FMDHTubHT4k8JfiCtOZe8C+B7wUEo1A74GYoH37RWUEKIWmzcPRowwBvr55hto2NDREQkhLkFlEwCltT4N3A28o7UeArS3X1hCiFrp9ddh/Hi4805Yuxa8vBwdkRDiElU6ATDvBhiO0fsfwNk+IQkhah2tYepUeOopY1jfzz4DDw9HRyWEuAyVHQdgAvB3YKXWeqdSqjUQb7+whBC1htYweTLMnAmjR8OCBeAs+b8QV7pKJQBa6+8x+gEUP98LPG6voIQQtURRETzyCMyfD3/7G8yaBU6OmERUCFHdKhoHYIFSqkM56+orpcYopYbbJzQhhEMVFsLIkUbh//e/w5tvSuEvxFWkohqAt4HnzSQgEcjEGAAoBPABFgNL7RqhEKLm5eUZ9/h//jm8+qqRAAghrioXTQC01r8CMUopLyCSv4YCTtZap9ZAfEKImnb6NAwaZAzuM3u2UfUvhLjqVLYPwElgo31DEUI43IkTxrj+//d/sGgRjBnj6IjEFW7btm1NXFxcFgIR2HkKenEBG5BYWFj4QOfOnY+UXmnP2QCFEFeS7GxjXP9ff4VPPjFG+hPiMrm4uCxs2rSp1d/f/5iTk5N2dDx1ic1mU5mZmeGHDx9eCFwwb49kY0IIOHwYevSAHTtg5Uop/EV1ivD39z8hhX/Nc3Jy0v7+/scxal8uUKUaAKVUPXNEQCHE1WL/foiKgvR0Y3S/KJmtW1QrJyn8Hcd878u82K9UDYBS6ialVBKQYj6/Vin1TiX3dVZK/aKUWmM+b6WU+q9SardSaplSyq1yL0MIUe127YJu3SAz0xjXXwp/IeqMyjYBvAHcDmQDaK1/A7pXct8JQHKJ5/8G3tBatwWOAWMreRwhRHVKTDQK/9OnjRn9unZ1dERCVLvU1FS3kJCQWjt3zYABA1pZLJaIkJCQ9kOGDLHk5eWpmjp3pfsAaK0PlFpUVNE+SqnmQD9goflcAbcBceYmHwB3VTYGIUQ12brVmM3P2Rk2bYKOHR0dkRB10vDhw4/u3bs3MTU1defZs2fVrFmz/Grq3JVNAA4opW4CtFLKVSk1mfOv6sszC3ga41YEgMZAjta60Hx+EGhW1o5KqXFKqa1Kqa2ZmZmVDFMIUaHNm+G228DHx/g9PNzREQlRI5KSktysVmv4pk2b6hUWFvLQQw81j4iIsIaGhoZPnz7dD8Bms/HQQw81DwkJaR8aGhq+YMGCRgBr1qzxjoyMbNejR4+2FoslYtiwYcFFRUUUFhYyePBgS/H206ZNa1KVmO69997jTk5OODk5ERkZeergwYM11ixe2U6A44E3MQrrP4GvgUcvtoNSKho4orXeppTqUdXAtNbzgfkAkZGR0oFEiOrw9ddw110QHAwbNkDz5o6OSNQhY8bQIjGRetV5zIgITi9eTOka6gv89ttv7kOHDm2zePHiP7p27Xrm9ddf92vQoEFRYmJi8pkzZ9T1118f1r9//xM//fRTvR07dngmJyfvTE9Pd+nSpYu1T58+JwF27NhR/5dffkkMDQ3N7969e8iSJUsatW3bNi89Pd11165dOwGysrIuaaasvLw8tWzZssYzZ86s8LVUl8oOBJSFMRVwVdwMDFBK3YkxfLAPRhLRUCnlYtYCNMdIKIQQ9vb553DvvWC1GolAkypdqAhxxTp69KjLXXfd1TYuLm5P586dzwJs2LDBJyUlpd7q1asbAeTm5jonJSV5bN682TsmJuaoi4sLLVq0KLzhhhtO/vDDD/UaNGhg69Chw6nw8PB8gJiYmKObN2/2io6OPnHgwAH3kSNHtujfv//xQYMGnbiUGEeOHBl84403nuzbt+/J6nvlF1epBEAp1Qr4G2ApuY/W+oKBBUqs+zvGFMKYNQCTtdbDlVIrgHuA/wAjgVWXGLsQorKWLjUm9omMhC+/hEaNHB2RqIMqc6VuD97e3kVBQUH58fHxXsUJgNZazZgxY//gwYPPK7DXrl3boLzjGN3Yzn/u7+9flJiYmLRy5Uqfd99913/ZsmW+K1asSCveprCwkIiIiHCAvn375syaNetQ6eM++eSTgVlZWS7r16/fc3mvtGoq2wfgcyANmAPMKPG4FM8Ak5RSuzH6BCy6xOMIISpj/nyIjTV6/H/zjRT+os5xdXXVX3755Z5PPvmk8bvvvusL0Lt37+Nz5871L+51v337dvcTJ044de/ePTcuLs63sLCQQ4cOuWzZssWrW7dup8BoAkhJSXErKioiLi7Ot1u3brnp6ekuRUVFjBo1Kue11177c8eOHec1cbi4uJCSkpKUkpKSVFbhP3PmTL/vvvuuweeff77X2fmSWg8uWWX7AJzVWs++1JNorTdiziWgtd4LdLnUYwkhqmDmTHjySbjzToiLA09PR0ckhEP4+PjY1q9fv7tHjx6h3t7eRRMnTsxKS0tz79Chg1VrrXx9fQvWrVu3JzY2NichIcHLarW2V0rpadOmHQwODi7cvn07ERERp8aPHx+clpbmcdNNN52IjY3N2bJli+fYsWMtNptNAbz00ksHqxLX008/3TIwMDAvMjLSChAdHX3s9ddfT7fHe1Ca0rri/nVKqWEYUwB/DeQVL9da/2y/0P4SGRmpt27dWhOnEuLqoDX885/w4otwzz1GE4CbjLlV1yiltmmtIx0Zw2+//ZZ27bXXZjkyhuqwZs0a7xkzZgTEx8fvdnQsVfXbb7/5XXvttZbSyytbA9ABiMW4h7/4lj5tPhdC1CZaw9NPw+uvG+3+CxeCi8z7JYQ4X2W/FYYArbXW+fYMRghxmWw2ePRRePdd4+fs2eAkc34Jcbmio6Nzo6Ojcx0dR3Wq7DdDItDQnoEIIS5TYSGMGmUU/s88A/56WmoAACAASURBVHPmSOEvhChXZWsAGgIpSqn/cX4fgHJvAxRC1KC8PBg2DD77DF55BZ591tERCSFqucomAC/aNQohxKU7fRruvhvWr4dZs2DCBEdHJIS4AlR2JMBN9g5ECHEJTpyA/v2NMf0XLoSxMrmmEKJyLtpAqJT6wfyZq5Q6UeKRq5S6pOEOhRDV5OhR6NULEhLg44+l8BeimkyZMqVpdR1r0qRJQS+88ELApe6fkJDged1114W1bdv2vMmJqkNFPYTqA2itvbXWPiUe3lprn+oKQghRRRkZ0KMH/PYbfPopDB3q6IiEuGrMnj07sCrb22w2ioqK7BKLl5eX7cMPP/xj9+7dO7/++utdzz77bItLnXCotIoSAJmFT4ja5sABY1jfPXtg7VoYIH1xhShPamqqW6tWrdoPGDCgVevWrdv37du3dW5urtPq1au9e/Xq1aZ4u5UrV/r07t27zSOPPNIsLy/PKSwsLHzAgAGtAKZOnRoQEhLSPiQkpP1LL73UpPi4FoslYtCgQZbQ0ND2e/bscYuLi/MJDw+3tmvXLrxr166hxcdOTk727NKlS7vmzZt3ePnll6s0C9c111yT16FDhzwAi8VS4OvrW5ienl4tA3tUdJAmSqlJ5a3UWs+sjiCEEJW0e7dR7X/smDGj3803OzoiIaqkSxfalV52990cnTKFzNxcnKKiCCm9/v77yXr8cbLT03EZOJA2Jddt2UJqRedMS0vzmDdvXlqfPn1ODRkyxDJ9+nT/qVOnZkyYMCH40KFDLkFBQYWLFy9uPHr06Kxhw4Ydf//995ukpKQkAWzevLnexx9/3Hjbtm3JWms6d+5sjYqKyvXz8yvav3+/+6JFi/6IiopKO3TokMtjjz1m2bhxY0pYWFh+RkbGuav03bt3eyQkJKTm5OQ4W63WiKeeeirT3d29yhfY8fHx9QoKClR4eHhexVtXrKIaAGfAC/Au5yGEqCk7dxpX/idPQny8FP5CVFLTpk3z+/TpcwogNjY2OyEhwcvJyYmYmJjsBQsW+GZlZTn//PPPXkOGDDleet+NGzd63XnnnTk+Pj62Bg0a2Pr163csPj7eGyAwMDA/KirqlLld/S5duuSGhYXlAwQEBJxrE+jTp0+Op6enDgwMLPT19S04ePBgla/g9+3b5zp69OjWCxYsSKuuSYMqCiJda/1StZxJCHHptm2D2283xvP//nsID3d0REJckotdsXt7Y7vY+sBACitzxV9aWdP4Ajz88MPZ/fr1a+vh4aH79+9/zNXVtUrHrVevnq3iraDk1b6zszOFhYXnBbRkyZKGr776ahDA/Pnz07p373665PqjR4863XHHHW1ffPHFP4sTjupQUQ2AqmC9EMLefvgBbrsNvLyM2/2k8BeiStLT0902bNhQH2Dp0qW+N91000kw2tQDAgIKZsyYEThu3LhzExa5uLjo4mmCe/bseXLdunUNc3NznU6cOOG0bt26Rj179rxgSOAePXqc2rJli3dKSoobQMkmgIqMGDEip3jK4NKF/9mzZ1W/fv3aDh06NHv06NHHLu0dKFtFCUBUdZ5MCFFFGzYYV/5NmxqFf5s2Fe8jhDiPxWI5O2fOnCatW7dun5OT4zJ58uTM4nVDhw7NDgwMzO/UqdPZ4mXDhw/PtFqt4QMGDGh1yy23nB42bFh2p06drJ07d7bGxsZm3nzzzWdKnyMoKKhw9uzZaYMGDWrbrl278EGDBrWujtgXL17c6H//+5/Xxx9/7BcWFhYeFhYWnpCQUC3zeldqOuBLOrBSHsD3gDtGU0Oc1vpFpVQr4D9AY2AbEFvRJEMyHbCok1atgpgYCAszOvwFXPKtxKKOkumAjd760dHRIbt27dpZ1voRI0YEd+zY8fTEiROv+CmLy1PedMD2nCkkD7hNa30tcB3QVyl1I/Bv4A2tdVvgGCCjlwhR2iefwODBcN11Roc/KfyFqHbt27e3JiUleY4fPz7b0bE4gt0mCddG1cJJ86mr+dDAbcAwc/kHwFRgrr3iEOKKs3AhjBsH3bvDF1+At9xwI8SlateuXX55V/87d+5Mrul4ahO7zhWqlHJWSv0KHAG+AfYAOVrrQnOTg0Aze8YgxBVl1ix48EGj3X/dOin8hRB2Y9cEQGtdpLW+DmgOdAHCKruvUmqcUmqrUmprZmZmxTsIcSXTGl5+GSZONGb2+/xzqFfP0VEJIa5idk0Aimmtc4B4oCvQUClV3PTQHPiznH3ma60jtdaR/v7+NRGmEI6hNUyZAs8/D7GxsGwZuLs7OiohxFXObgmAUspfKdXQ/N0T6A0kYyQC95ibjQRW2SsGIWo9mw0efRT+3/+Dhx+G998HF7t1zRFCiHPsWQMQCMQrpbYD/wO+0VqvAZ4BJimldmPcCrjIjjEIUXsVFsLo0TB3Ljz1FLz9NjjVSKWcEHVGamqqW0hISHtHx1GeV1991T84ODhCKdW55CQ/NpuNUaNGtQgODo4IDQ0N/+GHH6q9TdCedwFsBzqWsXwvRn8AIequ/HwYNsyYyvef/4R//AOUDLwpRF1z6623nhw8ePDx22677bxJklasWNFg7969HmlpaYnx8fH1H3nkkeDt27enVOe55XJDiJp25gzcdZdR+M+cCc89J4W/EDUgKSnJzWq1hm/atKleYWEhDz30UPOIiAhraGho+PTp0/3AuPJ+6KGHmoeEhLQPDQ0NX7BgQSOANWvWeEdGRrbr0aNHW4vFEjFs2LDgoqIiCgsLGTx4sKV4+2nTplVput+bb775TLt27S4YDG/VqlUNhw8fnu3k5ERUVNSpEydOuOzbt69qkxVUQBobhahJubnQv78xoc/8+cYtf0LUEWNWjWmReCSxWquyI5pEnF48cPGBirb77bff3IcOHdpm8eLFf3Tt2vXM66+/7tegQYOixMTE5DNnzqjrr78+rH///id++umnejt27PBMTk7emZ6e7tKlSxdrnz59TgLs2LGj/i+//JIYGhqa371795AlS5Y0atu2bV56erpr8VgDWVlZ1TJVX3p6uqvFYjmXGAQGBubv27fPtWXLlgXVcXyQGgAhas7Ro9CrlzG5z9KlUvgLUUOOHj3qctddd7X96KOP9nbt2vUMwIYNG3yWL1/eOCwsLLxjx47WY8eOuSQlJXls3rzZOyYm5qiLiwstWrQovOGGG04Wt7936NDhVHh4eL6LiwsxMTFHN2/e7BUWFpZ34MAB95EjR7aIi4vzadSoUdHFo6k9pAZAiJqQkQF9+kBKilH1P3CgoyMSosZV5krdHry9vYuCgoLy4+PjvTp37nwWQGutZsyYsX/w4MEnSm67du3aBuUdp6xphf39/YsSExOTVq5c6fPuu+/6L1u2zHfFihVpxdsUFhYSERERDtC3b9+cWbNmHapMzIGBgQVpaWluxc/T09PdqvPqH6QGQAj7O3jQGNZ3925Ys0YKfyFqmKurq/7yyy/3fPLJJ43fffddX4DevXsfnzt3rn/xtL/bt293P3HihFP37t1z4+LifAsLCzl06JDLli1bvLp163YKjCaAlJQUt6KiIuLi4ny7deuWm56e7lJUVMSoUaNyXnvttT937NhxXhOHi4sLxVP9VrbwBxgwYEDO0qVLG9tsNr799tv63t7eRdWdAEgNgBD2tGePUe1/9CisXw+33OLoiISok3x8fGzr16/f3aNHj1Bvb++iiRMnZqWlpbl36NDBqrVWvr6+BevWrdsTGxubk5CQ4GW1WtsrpfS0adMOBgcHF27fvp2IiIhT48ePD05LS/O46aabTsTGxuZs2bLFc+zYsRabzaYAXnrppYNVievll19uMmfOnKbZ2dmu1157bXjPnj2PL1u2bF9MTMzxtWvXNmjZsmWEp6enbeHChWnV/Z7YbTrg6iTTAYsrUlKSUfjn5xuFf+fOjo5I1DEyHXD1WbNmjfeMGTMC4uPjdzs6lqpyxHTAQtRdP/8Mt95qDPO7aZMU/kKIWkcSACGqW0IC9OxpTOazeTO0r7WDkAkhKik6Ojr3Srz6vxhJAISoTt9+C717Q0CAUfi3bevoiIQQokySAAhRXb74Avr1g9atjYF+goMdHZEQQpRLEgAhqsOyZXD33dChA2zcCE2bOjoiIYS4KEkAhLhcixfDffdB165GE0Djxo6OSAghKiQJgBCXY/ZsGDvWGOXvq6/Ax8fREQkhLtOUKVOqrQpv0qRJQS+88ELA5RyjW7duId7e3tf17NnzvE5FKSkpbtdcc01YcHBwRL9+/VqfPXu2SrOKSQIgxKV69VWYMAEGDYJVq4xe/0KIK97s2bMDq7K9zWajqMh+UwBMnjz58Lx58/4ovXzSpEnNH3vssYz9+/cnNmjQoPDNN9/0q8pxJQEQoqq0hr//Hf7xD7j/fli+HNzdHR2VEKIMqampbq1atWo/YMCAVq1bt27ft2/f1rm5uU6rV6/27tWrV5vi7VauXOnTu3fvNo888kizvLw8p7CwsPABAwa0Apg6dWpASEhI+5CQkPYvvfRSk+LjWiyWiEGDBllCQ0Pb79mzxy0uLs4nPDzc2q5du/CuXbuGFh87OTnZs0uXLu2aN2/e4eWXX67SdMEAAwcOzPXx8bGVXGaz2fjxxx+9R48efQxgzJgx2V988UXDqhzXbkMBK6VaAEuAAEAD87XWbyqlfIFlgAVIA2K01sfsFYcQ1cpmg8cfh7ffhocegnfeASfJo4WorC4LurQrvexu691Hp9wyJTM3L9cpaklUSOn1919zf9bjNzyenZ6b7jLwPwPblFy35cEtqRWdMy0tzWPevHlpffr0OTVkyBDL9OnT/adOnZoxYcKE4EOHDrkEBQUVLl68uPHo0aOzhg0bdvz9999vkpKSkgSwefPmeh9//HHjbdu2JWut6dy5szUqKirXz8+vaP/+/e6LFi36IyoqKu3QoUMujz32mGXjxo0pYWFh+RkZGeemBd69e7dHQkJCak5OjrPVao146qmnMt3d3S9rGN6MjAwXb2/vIldXVwAsFkt+RkaGWwW7ncee31yFwJNa63DgRuBRpVQ4MAX4VmsdAnxrPheidtPaGM63Vy+j8H/ySZg7Vwp/Ia4ATZs2ze/Tp88pgNjY2OyEhAQvJycnYmJishcsWOCblZXl/PPPP3sNGTLkeOl9N27c6HXnnXfm+Pj42Bo0aGDr16/fsfj4eG+AwMDA/KioqFPmdvW7dOmSGxYWlg8QEBBwrk2gT58+OZ6enjowMLDQ19e34ODBg7ViHh67BaG1TgfSzd9zlVLJQDNgINDD3OwDYCPwjL3iEOKy5OXB0qUwcybs3AmBgfDWW/DII6Cq1N9GCMHFr9i93b1tF1sf6B1YWJkr/tLKmsYX4OGHH87u169fWw8PD92/f/9jxVfTlVWvXj1bxVtByat9Z2dnCgsLzwtoyZIlDV999dUggPnz56d17979dEXHDAgIKMzNzXUuKCjA1dWVtLQ0t4CAgPyqxF8jly9KKQvQEfgvEGAmBwCHMZoIytpnnFJqq1Jqa2ZmZk2EKcRfsrLg5ZehZUujl7+zM3zwAaSlwaOPSuEvxBUkPT3dbcOGDfUBli5d6nvTTTedBLBYLAUBAQEFM2bMCBw3bty5CYtcXFx08TTBPXv2PLlu3bqGubm5TidOnHBat25do549e+aWPkePHj1ObdmyxTslJcUNoGQTQEVGjBiRUzxlcGUKfwAnJyduvPHG3Pfee68RwOLFixtHR0fnVPacUAMJgFLKC/gUeEJrfaLkOm1MRVhmO4jWer7WOlJrHenv72/vMIUwpKbCww8bo/g9/zx06gTffAO//gojRoBblZrYhBC1gMViOTtnzpwmrVu3bp+Tk+MyefLkc1eVQ4cOzQ4MDMzv1KnT2eJlw4cPz7RareEDBgxodcstt5weNmxYdqdOnaydO3e2xsbGZt58881nSp8jKCiocPbs2WmDBg1q265du/BBgwa1rq74O3fu3C42Nrb1jz/+6BMQEHDNp59+6gMwY8aMg3PmzGkaHBwccezYMZcJEyZUadZFu04HrJRyBdYA67XWM81lqUAPrXW6UioQ2Ki1vqBTSEkyHbCwK62NoXtnzDCG83V3N3r3T5woE/mIK5pMB2z01o+Ojg7ZtWvXzrLWjxgxIrhjx46nJ06ceMVPWVye8qYDtuddAApYBCQXF/6m1cBI4F/mz1X2ikGIiyoogBUrjPb9bdvAzw9eeMFo3w+4rHE7hBBXgPbt21s9PT1t8+bNO+DoWBzBnj0RbwZigR1KqV/NZc9iFPzLlVJjgX1AjB1jEOJCx4/D/PnGKH4HD0K7djBvHsTGgqeno6MTQlSjdu3a5Zd39b9z587kmo6nNrHnXQA/AOX1lIqy13mFKFdaGrz5JixcCCdPQs+e8O67cMcdcjufEKLOqRX3IgphV//9r9G+/+mnRkF/770waZLRwU8IIeooSQDE1amoyBiff8YMSEiABg1g8mT429+geXNHRyeEEA4nCYC4upw8Ce+9B7Nmwd690KqVUe0/Zgx4eTk6OiGEqDWk4VNcHf7805igp0ULY6z+gACIi4Ndu4znUvgLUSelpqa6hYSE1Nr7eQcPHmxp1qxZh7CwsPCwsLDwhIQETzAm+xk1alSL4ODgiNDQ0PAffvih2qcblRoAcWX79VfjNr7//Meo9r/7bqN9v2tXR0cmhBCV8vLLLx8sntWv2IoVKxrs3bvXIy0tLTE+Pr7+I488Erx9+/aU6jyv1ACIK4/NBuvWQVQUdOwIn31mjN63a5dxX78U/kKIMiQlJblZrdbwTZs21SssLOShhx5qHhERYQ0NDQ2fPn26HxhX3g899FDzkJCQ9qGhoeELFixoBLBmzRrvyMjIdj169GhrsVgihg0bFlxUVERhYSGDBw+2FG8/bdq0Kk/3W5ZVq1Y1HD58eLaTkxNRUVGnTpw44bJv376qTVZQAakBEFeOs2fhww/hjTcgORmaNYN//xvGjYOGVZoGWwjhCGPGtCAxsXqrsiMiTrN4cYUD+fz222/uQ4cObbN48eI/unbteub111/3a9CgQVFiYmLymTNn1PXXXx/Wv3//Ez/99FO9HTt2eCYnJ+9MT0936dKli7VPnz4nAXbs2FH/l19+SQwNDc3v3r17yJIlSxq1bds2Lz093bV4rIGsrKxKzwFQbNq0ac1ee+21wG7duuW+9dZbBz09PXV6erqrxWI5N7lPYGBg/r59+1xbtmxZUNXjl0dqAETtd+QITJtmjM8/bhx4eMBHHxmd/J5+Wgp/IcRFHT161OWuu+5q+9FHH+3t2rXrGYANGzb4LF++vHFYWFh4x44drceOHXNJSkry2Lx5s3dMTMxRFxcXWrRoUXjDDTecLG5/79Chw6nw8PB8FxcXYmJijm7evNkrLCws78CBA+4jR45sERcX59OoUaOii0dzvpkzZ/65d+/exN9++y352LFjzs8//3xTe7wHZZEaAFF7JScbV/tLlhjT8kZHw5NPwq23ymx84qpVUFTA7qO7ScpMIjkrmYk3TqS+W31Hh1U9KnGlbg/e3t5FQUFB+fHx8V6dO3c+C6C1VjNmzNg/ePDg8yapW7t2bYPyjlPWtML+/v5FiYmJSStXrvR59913/ZctW+a7YsWKtOJtCgsLiYiICAfo27dvzqxZsw6VPEbxFb2np6ceM2ZM9owZMwIAAgMDC9LS0s7NPpaenu5WnVf/IAmAqG20hvh44/79deuMq/1Ro+CJJyAszNHRCVFtThecJjUr9VxBn5yVTFJmEruP7qbQVnhuu/6h/bm26bUOjPTK5+rqqr/88ss9PXv2DPHy8rKNHz/+aO/evY/PnTvXPzo6Otfd3V1v377d3WKxFHTv3j13wYIF/o899lj2kSNHXLZs2eI1e/bsA9u3b/fcsWNH/ZSUFLeQkJD8uLg43wceeCAzPT3dxd3d3TZq1Kic9u3bn42NjT1vFkAXFxdSUlKSyoutuFrfZrPx2WefNbRarWcABgwYkPPOO+80efDBB4/Gx8fX9/b2LpIEQFyd8vNh2TKjR/+vv0KTJka1/8MPg0wHLa5gOWdzSM5MvqCg35ezD23Ohu6snGnr2xarv5W7w+7G6m/F6mclzC/s6rn6dzAfHx/b+vXrd/fo0SPU29u7aOLEiVlpaWnuHTp0sGqtla+vb8G6dev2xMbG5iQkJHhZrdb2Sik9bdq0g8HBwYXbt28nIiLi1Pjx44PT0tI8brrpphOxsbE5W7Zs8Rw7dqzFZrMpgJdeeulgVeK69957Wx09etRFa63Cw8NPL1myZB9ATEzM8bVr1zZo2bJlhKenp23hwoVp1f2e2HU64Ooi0wFfxY4dMybimTMHDh2C8HDjNr7hw42rfyGuAFprMk5lkJz5VwGfnJVMcmYy6SfTz23n7uxOmF8YVn8r4X7h5wr6kMYhuDm7XeQMl0amA64+a9as8Z4xY0ZAfHz8bkfHUlU1Ph2wEBe1Z48xQt/ixXDqFPTqBYsWwe23S/u+qLVs2sb+4/vLLOiPnf3rNm5vN2/C/cO5ve3tWP2shPuHY/WzYmlowdmpyp3EhbALSQBEzUpIMNr3V64EFxcYNgwmToRrpY1T1B4FRQXsObbngoI+JSuF0wWnz23nX88fq7+Ve9vfa1zVmwV9kHfQBR3GxJUtOjo6Nzo6OtfRcVQnSQCE/RUWGgX+zJnw00/QqBFMmQKPPQZBQY6OTtRhZwrOkJqdekFBvyt7FwW2v/pbtfBpgdXfyrhO485V21v9rfjV83Ng9EJcHrslAEqpxUA0cERrHWEu8wWWARYgDYjRWh8r7xjiCpeba1Trv/kmpKVBmzbw1ltGr/760rFJ1JwTeSfK7Ij3x7E/znXEc1JOtGnUBqu/lQGhA87riOft7u3gVyBE9bNnDcD7wFvAkhLLpgDfaq3/pZSaYj5/xo4xCEc4cABmz4b58+HECbjlFuN+/v79wVnaP4V9aK3JPJ1ZZkF/KPevW6/dnN1o17gdkUGRjLhmxHkd8TxcpOOpqDvslgBorb9XSllKLR4I9DB//wDYiCQAV4+ffzba95cvN+7nv+ceo0d/ly6OjkxcRbTWHDhxoMyC/uiZo+e283LzwupnpVfrXud1xGvVqBUuTtL6KURN/xcEaK2L74k5DATU8PlFdbPZYO1ao+DftAm8vY3pdx9/HFq2dHR04gpWaCtk77G9FxT0yZnJnCo4dW67xp6NCfcP5x7rPed1xGvu01w64olLMmXKlKb/+te/DlfHsSZNmhTk5eVV9NJLL2Vc6jGcnZ07h4SEnAEICgrK/+6773YDpKSkuMXExLTOyclx6dChw+lPP/30Dw8Pj0rf2++wNFhrrZVS5QaqlBoHjAMIDg6usbhEJZ0+bQzR+8Yb8Pvv0KIFvP46PPAANCh3JE0hLnC28Cy/Z/9+QUe837N/J7/o3FwoNPNuRrh/OGM7jj2voPevLwNFieo1e/bswKokADabDa01znZq4nR3d7eVNZrgpEmTmj/22GMZ48aNOzZs2LDgN9980++ZZ57JrOxxazoByFBKBWqt05VSgcCR8jbUWs8H5oMxEFBNBSgqcPgwvP02zJ0L2dkQGQmffAKDB4Nrtc5UKa4yuXm5567gSxb0e4/txaZtACgUrRu1Jtw/nDvb3nmuoA/zC8PH3cfBr0BciVJTU9369u0b0qFDh9OJiYn1QkNDz6xYsSItPj6+/uzZs5ts2LBhD8DKlSt93nnnHf+QkJCzeXl5TmFhYeGhoaFnVq9e/cfUqVMDli5d6gcQGxub+cILLxxJTU11u/3220M7dux4cseOHfXXrVu3a/v27R4vvPBCs6KiIuXr61v4448//g6QnJzs2aVLl3aHDh1yGz9+fMZzzz1XbtlXWTabjR9//NF71apVewHGjBmTPXXq1KDanACsBkYC/zJ/rqrh84tLlZhoXO1/9BEUFMCAAcbEPLfcIgP3iPNknc4yCvdSBf3BE3+NkOrq5Epo41Cua3odwyKGneuIF9o4FE9XTwdGL+yuS5d2Fyy7++6jTJmSSW6uE1FRIResv//+LB5/PJv0dBcGDmxz3rotW1IrOmVaWprHvHnz0vr06XNqyJAhlunTp/tPnTo1Y8KECcGHDh1yCQoKKly8eHHj0aNHZw0bNuz4+++/36T4invz5s31Pv7448bbtm1L1lrTuXNna1RUVK6fn1/R/v373RctWvRHVFRU2qFDh1wee+wxy8aNG1PCwsLyMzIyzlUH7N692yMhISE1JyfH2Wq1Rjz11FOZ7u7ulb6wzc/Pd4qIiLA6OzvryZMnH46Njc3JyMhw8fb2LnI1L7wsFkt+RkZGlYaTtOdtgJ9gdPjzU0odBF7EKPiXK6XGAvuAGHudX1QDrWHDBqN9f/168PQ0qvgnTIDQUEdHJxxIa82fuX+WWdBnnf5r1Nf6rvUJ8wujh6XHeUPftvFtIx3xRI1p2rRpfp8+fU4BxMbGZs+ePbuJk5NTRkxMTPaCBQt8H3300eyff/7Z67PPPvuj9L4bN270uvPOO3N8fHxsAP369TsWHx/vPWTIkJzAwMD8qKioU+Z29bt06ZIbFhaWDxAQEHBuWuA+ffrkeHp6ak9Pz0JfX9+CgwcPurRp06bSE/vs2rVre6tWrQqSkpLcevfu3a5Tp05nfH19qzTtcFnseRfAfeWsirLXOUU1ycszqvVnzoQdO6BpU3jlFXjoIWjc2NHRiRpUZCvij5w/LijoU7JSyM3/a1C0Rh6NCPcP5652dxlt82ZB36JBC5yUkwNfgah1LnbF7u1tu+j6wMDCylzxl1bWNL4ADz/8cHa/fv3aenh46P79+x9zrWIzZr169WyV2a7k1b6zszOFhYXnBbRkyZKGr776ahDA/Pnz07p373665PpWrVoVAISHh+ffeOONuVu2bKk3cuTIY7m5uc4FBQW4urqSlpbmFhAQOD7WUgAAE0tJREFUkE8VSAou/pKdDe++awzWc/gwdOgA770H990H7u6Ojk7YUV5hHruO7jqvx31SZhK/Z/9OXlHeue0CvQIJ9w9n5LUjz+uI16R+E+lxL2qt9PR0tw0bNtTv1avXqaVLl/redNNNJwEsFktBQEBAwYwZMwK/+uqr34u3d3Fx0Xl5ecrd3V337Nnz5JgxYyz//Oc/D2utWbduXaP3339/b+lz9OjR49SkSZNapqSkuBU3AZSsBbiYESNG5IwYMSKnrHWZmZnOXl5eNk9PT52enu6ydetWr2efffawk5MTN954Y+57773XaNy4cccWL17cODo6usxjlEcSAAG7dsGsWUZhf+aMMSHPkiXGBD3ypX5VOZl/kpSslAturdtzdA9F2viuUigsDS3GZDZtbj+vI15Dj4YOfgVCVJ3FYjk7Z86cJuPGjasXEhJydvLkyec6yg0dOjT77bffdunUqdPZ4mXDhw/PtFqt4REREadXr179x7Bhw7I7depkBaMT4M0333wmNTX1vPb2oKCgwtmzZ6cNGjSorc1mo3HjxgUJCQm7Ljf2X3/91ePRRx9tqZRCa80TTzxxuHPnzmcBZsyYcfDee+9t8/LLLzdr37796QkTJlRp1kWZDriu0hp++MFo31+92ujBf//9xsQ8ERGOjk5cpuzT2ed63Jcs6Pcf339uGxcnF0J8Q85dxRcX9KGNQ6nnWs+B0YvqItMBG3cBREdHh+zatWtnWetHjBgR3LFjx9MTJ0684qcsLo9MBywMhYUQF2cU/Fu3Gm36zz0HjzxitPWLK4bWmkO5h8os6I+c+usuI08XT8L8wrgl+JbzOuK19W2Lq7Pcuinqrvbt21s9PT1t8+bNO+DoWBxBEoC64vhxWLjQGKN//36jF//cuTBiBNSTq73arMhWxL7j+8rscX8i78S57Rp6NMTqZyU6JPq8jngtG7aUjniizmrXrl1+eVf/O3fuTK7peGoTSQCudvv2GYX+ggXG7Hy33mp08uvXD5ykUKhN8ovy2X109wUFfWp2KmcLzzVP0tSrKVY/K/d3uP+8gr6pV1PpiCdqI5vNZlNOTk61v735KmSz2RRQ5t0KkgBcrbZsMW7ji4sznt97rzExT+fOjo1L8P/bu/fguKr7gOPfn1bSaqWVbWkljLHBlm2wLTzgB8MrjwkGA8YMlAmdmjQToHiYkiENgaZAQ1vSaYG+MiXTTDLEEEKaAgnJEJMhA26AMjhAwMYOxtigAAYbC8uSLVvSSqtd/frHOZKu5JWlXaRdaff3mbmz95779lnrd+65Z8/pTHSyu3X3MYG+qa1poCEewNzpc2msb+TChguHBPqaSE0er96YjO1oaWlprK+vb7dCQG719fVJS0vLdGBHuvVWACgkqRQ89ZR7v//SS65P/ltvha99zfXVb3LqUPzQYHV9INDvad8zsE1IQiysXUhjfSNfXPLFgYZ4i2KLqCqvyuPVGzM+ksnk+ubm5g3Nzc1LAat2zK0+YEcymVyfbqX9CqAQdHbCww+7n/I1NblR+G65BW64wY3OZyaMqtLc0Zy2IV5zx+BYIhWlFSyKLTqmxf3C2oWUhzLqvdNMcqpurKyODjcdPTo4n255tG02b4b587O7lsnwKwAzeVkNwFSlCu+9Bw8+6DrvOXQIzjkH7rkHrroKSi1rx0N3spvWrlZa460c7DpIa1crH7Z/OCTQH+4e7HtjWngaS+qWsGbhmiGBfu70uYRKJmakMJM9Vdf1RTaBeaR9OjrcccciFIJo1JXTo9HBac6cwTTrg8tMFIsSn0JnRw+bNz7HxV9aM7En6g/2W7YMTlu3uqAv4gL+bbfB+edP7HVMcV29XQNB/GDXwSFBfchyvHUgLTjufNAJVSewpG4J605fN/B+vrG+kVnRWdYQb4KoQnf3+AXq/rSxBuuSkmMDdTQKs2cPXR6+Tbp9gsHdvi4mX6wAkKWnf/g4y77xJZap0rG2g+j0cfopXV+fq8bfunVosG9vd+vLyuCMM+Dqq12DvtWrs68fnKJUlY5Ex8gBvKuVg/Fj04Mt6YebUTGDWCRGXWUdJ0ZPZOkJSweWBz4rY8QiMU6qPolYpY2JcDz9wTrTQD3aNn1j6nndBet0QXfWrOwCdTQKFRUWrE1hsQLAGG15YStb7v17SupPZP1/b+DcK1bzu/+YQ8eadVxVnmW7llTKdcMbDPRvvAFH/G+7w2EX7K+5BlascAF/6VIoL5x3xqrKkZ4joz6RD09PpNKPeSEINZGagcB9yvRTWD5rOXURF8CHB/S6yjpqI7VFPTKdqhv/aTwDdUeH+3qPhUj6oDtzJixYkHmgjkbdwJUWrI05vuL9qzcG+z9s4am7/oH5//czLviolZUKv1rSAEDtzFou37VnlCMEpFKwa9fQJ/tt29xfSnCPF2ee6brjXbnSBfzTT3dP/FNEn/ZxuPvwQLAeS0BvjbeS7EumPV5IQtRGageC9YLaBZw9++y0T+X98zUVNQX9rl0VEonxDdQdHa6DyLFKF3Dr611FVKaBurragrUx+WIFgGFSyT5Cpe6J/o1VZ3LjH/fzwbQQj37us5x6211cecUlox8kmYS33x76ZL9tm2saDO4v3rJlcN11LtivXAlLlkyqhnupvhRt8baMqtnb4m30afo62tKS0iGBe3Hd4uM+lcciMaZXTJ/yPdj1B+vxfG+dabAePtXXQ0ND5oG6/8na+o8ypjBMnoiTZ5t+spF937+XC3a8RtuTm1m+6hz06//Exngna2+9iS+PFJwTCdi5c+iT/fbt7gUoQFUVLF8O69cPBvvFi13z3xzpTfXSFm/LqJr9cPdhlPSto8Kh8JBgfcbMM0YM4v3z1eXVeWkcpwq9vS6bxjqNtn1/q/GxPHX39o79Wquqjg26sZj7VWemgToadT08W7A2xowkLwUAEbkUuB8IARtU9b5cnr//Kf/N321j+19dT+NHu1l9IE5S4IV59ZQe+AT6+lj7Z5fDxx/Ds8/Cvn1ufvjngcFBV6iudsH+ppsGq/FPOy3rYK+q9KR6iPfGiSfjdCe7B+bjvW65vad91IDe3tM+4jkqyyqHBO55M+albfwWXK4IVZJMyvGDaBv0JOCDBLyTQfAdaxAe676ZBOBMVFYeG3Rralx/S5kG6mjUBX8L1saYXMp5R0AiEgLeAVYDe4HXgGtUdedI+2TbEVCiO8GmBx/j4EvPE3l3G7Ob97DgUDtblqxg7bfvZt8bb1F39+0cqCznaN3JnNwwn2i8wwX3/fuRNNEjWR8jeeIJJGbW0XNiPT31NRydN4uDi+dycHYtXake4oluOhNxuhJxunrdZ7y3m65hwbs72R/Y4/SkuulOxelOxelJueVEX/eIT+HpVEg1lRKjkjoiGiOidYRTMcpTdZQlY5Ql6ihNxCjpqaOkO4bEY6R6IhkH4EyqoDMh4to9lpePfSory2z78do3hxU4xmTNOgIyx5OPAsB5wN2qeolfvhNAVe8daZ9sCwDPnjaP1e/uYSwVz+1h+Lga9lX7z2lDlz+uhuYo9GZTZ9JXAskI9EYCnxXZpSUrhq7vmQZddRCvhdSxPYaEQhMfDMcrEFtQNWZ8WQHAHE8+XgHMBoJjL+8Fzhm+kYjcCNwIcMopp2R1op2NKzlKFy1VUZqn1dJREaEzHKazPExXeZjO8go3hSMkyiopoYyQlFFCKSHKCFFGiZQSpoyFUsqibrc+JKXuEzdfVhImXBKh3E/hkgrCoQgVfiovLSMUYshUUsIxaemm0bYLBt7hQbiszIKqMcaY9CZtI0BVfQB4AFwNQDbHuOXJX4zrNRljjDGFIh/NjvYBwaHp5vg0Y4wxxuRIPgoArwGnikiDiJQD64CNebgOY4wxpmjl/BWAqiZF5GbgGdzPAB9S1bdyfR3GGGNMMctLGwBVfRp4Oh/nNsYYY0x+XgEYY4wxJs+sAGCMMcYUISsAGGOMMUXICgDGGGNMEcp5V8DZEJEWYM8om9UBB3NwOZON3XdxsfsuLp/2vueqav14XYwpLFOiADAWIvJ6MfZ5bfddXOy+i0ux3rfJDXsFYIwxxhQhKwAYY4wxRaiQCgAP5PsC8sTuu7jYfReXYr1vkwMF0wbAGGOMMWNXSDUAxhhjjBkjKwAYY4wxRWjKFwBE5FIR2S0iTSJyR76vJxsicrKIPC8iO0XkLRH5uk+vFZFNIvKu/6zx6SIi3/X3/AcRWRE41rV++3dF5NpA+koRedPv810RkdzfaXoiEhKRN0Tk1365QURe9df6uB82GhEJ++Umv35e4Bh3+vTdInJJIH1Sfj9EZIaIPCEiu0TkbRE5rxjyW0S+4b/jO0TkURGpKNT8FpGHROSAiOwIpE14Ho90DmOOoapTdsINJ/xHYD5QDmwHGvN9XVncxyxghZ+vBt4BGoF/Be7w6XcA/+LnLwN+AwhwLvCqT68F3vOfNX6+xq/7vd9W/L5r8n3fgfu/Ffgf4Nd++WfAOj//A+AmP/9V4Ad+fh3wuJ9v9HkfBhr8dyI0mb8fwI+B9X6+HJhR6PkNzAbeByKBfL6uUPMb+DywAtgRSJvwPB7pHDbZNHya6jUAZwNNqvqeqiaAx4Ar83xNGVPV/aq61c8fBd7G/bG8Ehco8J9/4uevBB5R5xVghojMAi4BNqlqm6oeAjYBl/p101T1FVVV4JHAsfJKROYAa4ENflmAVcATfpPh993/7/EEcKHf/krgMVXtUdX3gSbcd2NSfj9EZDouODwIoKoJVT1MEeQ3bgjyiIiUApXAfgo0v1X1RaBtWHIu8nikcxgzxFQvAMwGPgos7/VpU5av5lwOvArMVNX9flUzMNPPj3Tfx0vfmyZ9MvhP4G+APr8cAw6ratIvB6914P78+na/fab/HvnWALQAP/KvPjaISBUFnt+qug/4d+BDXOBvB7ZQ+PkdlIs8Hukcxgwx1QsABUVEosAvgFtU9UhwnS/lF9RvNkXkcuCAqm7J97XkWCmuavj7qroc6MRV1Q4o0PyuwT2dNgAnAVXApXm9qDzKRR4X4vfIjJ+pXgDYB5wcWJ7j06YcESnDBf+fquovffInvqoP/3nAp49038dLn5MmPd8+A1whIh/gqmtXAffjqj9L/TbBax24P79+OtBK5v8e+bYX2Kuqr/rlJ3AFgkLP74uA91W1RVV7gV/ivgOFnt9Bucjjkc5hzBBTvQDwGnCqb0VcjmsotDHP15Qx/17zQeBtVf1OYNVGoL/V77XArwLpX/Eth88F2n2V3zPAxSJS45+2Lgae8euOiMi5/lxfCRwrb1T1TlWdo6rzcHn3nKr+OfA8cLXfbPh99/97XO23V5++zrcabwBOxTWQmpTfD1VtBj4SkUU+6UJgJwWe37iq/3NFpNJfV/99F3R+D5OLPB7pHMYMle9WiJ92wrWefQfX+vdb+b6eLO/hs7hquj8A2/x0Ge5952+Bd4H/BWr99gJ8z9/zm8BZgWP9Ba5RVBNwfSD9LGCH3+e/8L1ATpYJ+AKDvwKYj/uD3gT8HAj79Aq/3OTXzw/s/y1/b7sJtHifrN8PYBnwus/zJ3EtvAs+v4FvA7v8tf0E15K/IPMbeBTX1qEXV+tzQy7yeKRz2GTT8Mm6AjbGGGOK0FR/BWCMMcaYLFgBwBhjjClCVgAwxhhjipAVAIwxxpgiZAUAY4wxpghZAcAUNBFJicg2EdkuIltF5PxRtp8hIl8dw3FfEJGzsrymp0VkRjb7GmPMeLECgCl0cVVdpqpnAncC946y/QzcKHQTRlUvUzf4jzHG5I0VAEwxmQYcAjfugoj81tcKvCki/aPG3Qcs8LUG/+a3vd1vs11E7gsc709F5Pci8o6IfG74yURkloi86I+1o38bEflAROpE5C/9um0i8r6IPO/XXywiL/tr+7kfI8IYY8aVdQRkCpqIpHA9q1UAs4BVqrqlfzhaVT0iInXAK7guZefieiRc6vdfA/wdcJGqdolIraq2icgLwBZVvU1ELgNuVdWLhp37NqBCVf9ZREL+fEf92AdnqepBv10Z8BxuHPeXcX3kr1HVThG5Hdcz3j9O5L+TMab4lI6+iTFTWlxVlwGIyHnAIyKyFNf16j0i8nncUMSzST9s6kXAj1S1C0BVg+O79w/atAWYl2bf14CHfIB/UlW3jXCN9+P6uX/Kj5DYCGx2XbxTjisUGGPMuLICgCkaqvqyf9qvx/UZXw+sVNVe/1RekeEhe/xnijT/l1T1RV/AWAs8LCLfUdVHgtuIyHW4Woeb+5OATap6TYbXYowxGbE2AKZoiMhiIIQbUnY6cMAH/wtwQRjgKFAd2G0TcL2IVPpj1GZwvrnAJ6r6Q2ADbsjf4PqVwF8DX1bVPp/8CvAZEVnot6kSkdMyu1NjjBmd1QCYQhcRkf6qdwGuVdWUiPwUeEpE3sSNyrcLQFVbRWSziOwAfqOq3xSRZcDrIpIAngb+dozn/gLwTRHpBTpwQ7YG3QzUAs/76v7XVXW9rxV4VETCfru7cCPcGWPMuLFGgMYYY0wRslcAxhhjTBGyAoAxxhhThKwAYIwxxhQhKwAYY4wxRcgKAMYYY0wRsgKAMcYYU4SsAGCMMcYUof8HonQ5JHPyQO0AAAAASUVORK5CYII=\n", "text/plain": [ "
" ] @@ -359,22 +345,17 @@ "n_features = [2, 10, 50]\n", "max_batch_size = 100000\n", "\n", - "plot_absolute_time(results, n_features, max_batch_size=max_batch_size)" + "plot_absolute_time(experiments_eq, results, n_features, max_batch_size=max_batch_size)" ] }, { "cell_type": "code", "execution_count": 6, - "id": "ec9d0fbb", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiIAAAEWCAYAAABbt/wMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdd3hUZfbA8e9JQocgnSQQQoeAdFEhriCKLmJhFVZBVCygKCpY1sIiouvKz7WBqIggoigqi4qKK6KwVGEDCNKVIiV0aaGknt8f700cQsoEMkxCzud55knmzi1n7tyZOfNWUVWMMcYYY4IhJNgBGGOMMab4skTEGGOMMUFjiYgxxhhjgsYSEWOMMcYEjSUixhhjjAkaS0SMMcYYEzRFNhERERWRBrk8/paI/N3n/r0isltEEkWkymkcL8Y7ZtjpxlxQvOdQL9hxnC0iMlFEnsvH+rleG4WNiHwjIrcFOw5/iEi0d/2Fnsa2ZUTkSxE5JCKfBiI+c3pEpI+IzAx2HKZ4yjMREZEtIpIsIlWzLF/ufeDHePcnevevy7LeK97y2737t4tImvdhligim0XkXRFpVGDPClDVe1T1We+YJYCXga6qWl5V9wfyy8p7jvMLaF9zROQu32Xec9hUEPsvCIUhSRORi0VkYbCO7y8RGS4iH/guU9U/q+p7wYopN977//KM+6q61bv+0k5jdzcCNYAqqtqzAGIrKSJTvRhVRDpleVxEZKSI7PduI0VEfB5vJSJLReSY97fVmcYUDPn9vMnu/aqqk1W1a2AiNCZ3/paIbAZuzrgjIucDZbNZbwNwq896YUAvYGOW9RapanmgInA5cBxYKiLN/Q89Z9n8WqsBlAZWF8T+TcEqoATmamBGAeyn2AhC4lgH2KCqqfndMJdY5wO3ALuyeaw/cD3QEmgBXAMM8PZXEvgC+ACoBLwHfOEtDxgvOSqyJdHGBISq5noDtgBDgf/5LPsX8BSgQIy3bKK3fDdQyVvWHfgG92Fxu7fsdmB+Nsf5CpiaSxyPAjuBBOAO79gNfI79Ju6L6CguuZkIPAc08pYpkAj8AMz17h/1lv01m+OFes9nH7AJuM/bJsx7vCIw3otph3esUKApcAJI8/Z90Fu/lLe/rd45egso43O864CfgMO4xO0q4B/efk54+3rdW9f3uVcEJgF7gd+81yrE91x7xz2ASyj/nMdr/QSwxlv/XaC099gq4BqfdUt456a195wyzm8icDEuyR3qxbTHi7Git22Mt/6d3rZzveVxwELgILCNP66ZicAY4GvgCLAYqJ8l9mVAm2zOT5y3r07e/SbAd8DvwHqgl88+8jqXC4DXgUPAOqCLz7a3466TI9557pPN+b0KSAZSvPO0wls+B7gry3Fe8c7DJqCDt3ybdy5v89lnrtdVluP77ns/7pqtj3tP7Pdez8nAed767wPpuB8KicBjPq9dxvsgEpjunc9fgbtzOPYzWZ77nZzGNZLLtbs94zX2WbYQ6O9z/07gR+//rrj3rfg8vhW4Kof9zwH+CSzBvUe/ACr7PH4Rf1y7K3xj8bb9h3fujwMNvGXPedskAl8CVbzzfxj4H398tp50zn2vGXL+vLkaWO7taxswPMvzzPp+vR2fz2XcNfc/3LX+P6BDlmM/6z2fI8BMoKr3WGlccrffOxf/A2rk9T1jt+J9y3sF9+V0Oe5Duynuy3Y77tdN1kTkOeBt4F5v2Se4khR/EpE7gN05xHAV7kO2OVAO+JBTE5FDQEfch1vpjHi8x7N7I2dun8Mx78F92dQGKgOzOfkD+DNgrBdPddwH1ICcniPuw3+6t68KuA+ef3qPtffiv8KLPwpo4j02B+9LKrvYcR/eX3j7jMGVSt3pE0cKcLf3ut2LS+Qkh+e8BZdwZDznBT7n8DHgY591rwN+zuX83oH7YqoHlAemAe9nWX+Sd/7K4K6nI971UgL3odzK5/Xd752nMNyH9RSfY0Xg86WScX5w1802oL23vJx3v5+3n9a4L99YP89lKjDYi++v3mtW2dvvYaCxTzzNcjjHw4EPsizLfI19jtPPe82ew31xjMElHV2981Q+r+sqm2Nn7HuQ9/zLeOfpCm/f1XBJ+qtZ3/8+9096rb3138C951rhkrjL/Hnu5PMayeNzKrtE5BBwoc/9dsAR7//BwDdZ1v8KeDiH/c/BXWMZn0H/znguuPfrfqAb7v17hXe/ms+2W4Fm3nkv4S37FZcIVsQl/xtwn7Vh3vN+N5f3V9ZrJuvnTSfgfC+eFrjPz+tz2V/mPnDX0gGgrxfLzd79Kj7H3oj7kVfGu/+C99gA3DVYFnf9tgXC8/qesVvxvuWniPB9XLXLFcBa3JsyO5OAW0XkPOBS4HM/95+AewNkpxfuTblKVY/iPtCy+kJVF6hquqqe8POYuemF+0Depqq/434NASAiNXAfOg+p6lFV3YP7Qrgpux159dL9gcGq+ruqHgGe91n/TmCCqn7nxb9DVdflFaBXBXUT8ISqHlHVLcBLuA+QDL+p6jh1dfrv4b4ka+Sy29d9nvM/+KNK7gOgm4iEe/f74q6JnPQBXlbVTaqaiCtpuSlLEftw7/wdB3oDs1T1I1VNUdX9qvqTz7qfqeoSdcX6k3Ffehm6Af9RVfVZ1hOXKP5ZVZd4y7oDW1T1XVVNVdXluC+Unn6eyz24ayJFVT/GJedXe4+lA81FpIyq7lTVM6kG3OzFmAZ8jEsMR6hqkqrOxJUsNPDjuspOgqqO9p7/cVX91bvuklR1L64t1aX+BCkitXHJ/99U9YT3er2DT/VsHvJ7jeRXeVwykuEQUN47b1kfy3i8Qi77e9/nM+jvQC/vurkFmKGqM7z373dAPO66zDBRVVd75z3FW/auqm5U1UO4kuONqjrLu8Y/xSXKp0VV56jqz148K4GP8PN1xV3Tv6jq+168H+F+lF3js867qrrBe10+4Y/3YwruR0QDVU1T1aWqevh0n4cpHvJTR/w+7tdPXVyykS1VnS8i1XBVN1+p6nGf9mG5icIV72YnEljqc/+3bNbZ5s9B8iEyyz59j1kH96tmp89zC8klhmq4XwhLfdvK4X4xgPuiOZ32DVW9OHxj+w13LjNk1p2r6jHv+OVz2WfW5xzpbZsgIguAG0TkM+DPwIO57Ccym7jCODkJ8j1WbU5tS+TLtw3AMU5+Dt1wpWS+HgImqeoqn2V1gAtF5KDPsjDcte3PudyRJdn5DYhU1aMi8lfgEWC8d54e9ieZzMFun/+PA6hq1mXlyfu6ys5J16iXVL8GXIL7Eg7B/fr1RySQkQBl+A1X8uDv9vm5RvIrEQj3uR8OJKqqikjWxzIeP0LOsr43SuCumzq4ZNb3i7oErhQ1u20zZH1Ns3uNT4uIXAi8gCvBKYkr8fK3p1LW1wVy+Vzh5Pfj+7j38hTvx+gHwFM+yZcxp/C7RERVf8PVfXfDFaHm5gPgYXJJWLLRA5iXw2M7cRd3hujsQszHsfyR2zG3AUm4etHzvFu4qjbLIZZ9uA+WZj7rV1TXYDdjf/VziCO357UP9wukTpY4cyqt8kfW55zgc/893K+/nrgGxxnHyS7GhGziSuXkD1vf7XI7BznyekRdimv34asncL2I+CZL24D/+rwG56nrAXIv/p3LKDk5q848P6r6rapegStxWgeMyyHkgrxO87qu/Dn+896y81U1HPf6Si7r+0oAKouIbylCfq6//F4j+bUa11A1Q0v+aLC+GmiR5fVsQe4N2rO+N1Jwr8E2XGmJ73VVTlVf8Fn/TJ7HUe+vbweBmnns+0NclV1tVa2IazskuazvK+vrAn6+rl5p4TOqGotrZ9Id/0vITDGV39bbd+Lqf4/msd4oXBXO3NxWEpFQEakrIqNxdZrP5LDqJ8DtIhIrImWBp/MXdrZ24+qmc/IJ8ICI1BKRSsDjGQ+o6k5cA62XRCRcREJEpL6IZBR97gZqZbTAV9V03BfTKyJSHUBEokTkSm/98UA/Eeni7StKRJrkFadXdP8J8A8RqSAidYAhuETwdN3nPefKuFKtj30e+xxogysJ8U0y9+KqJnzj/AgY7L2+5XFfeB9rzj0mJgOXi0gvEQkTkSp+dqeMA1ZmU/ybAHQBHhSRe71lXwGNRKSviJTwbheISFM/z2V13DVRQkR64tpMzRCRGiJynYiUwyWoid75yM5uIKYgek74cV35owIu3kMiEoVrFJ413pyuv224xpb/FJHSItIC9xnh7/WX32vkFCJSSkRKe3dLenFkfOFOAoZ45yQS9+NoovfYHFwDzwe8fdzvLf8hl8Pd4vMZNALXuD4N93yvEZErvc+00iLSSURq+fs8cuNVme3wjh8qIndwctJ+0ueNpwKutOqEiLTHVX1myO796msG7n3S23sv/hWIxb1/ciUinUXkfK/K6jAuWcvpvWAMkM9ExKvPjPdjvd9V9fssxdi+LvaKRg/jPhDCgQtU9ecc9vcN8CruQ+JXcv+w8Ndw4D0ROSgivbJ5fBzwLa4F/DJOLQW6FVfkmdHDZCru1zBefKuBXSKyz1v2Ny/2H0XkMDALaOw9vyW4xomv4Oqp/8sfv0heA24UkQMiMiqbOAfhfjFtwjUK/hCY4N8pyNaHuCRrE66qJHMgMa8++N+46rlpPsuP4fUK8M7nRV4MGdV5m3Et+wfldFBV3YorbXsYV0X3Eyf/ms1Jjt12vX12AR4Xkbu8KoSuuDYUCbji5ZG4YmvI+1wuBhrifgX/A7hRVffj3kdDvH3+jiuhuZfsZRSP7xeRZX48v7zkeF356RlccnkI1ysp63X+T2Co97o+ks32N+MaPybgGnA/raqz/Dx2vq6RHKzHlQpF4d6vx/njvTMW13DyZ1wj7K+9ZahqMq5r76243h134BpzJudyrPdxicwuXOPcB7x9bcM13n4S9yW/DZfQFWQ33bu9fe7HNXr1HTMnu8+bgcAIETkCDMMl2XjxZvd+xefx/biSjIe94z0GdFfVfeStJu6z8DCuLeF/yb0tmTGZvQyMQUS24Fri5/hFIiLDgEaqestZCywXIrIGlxCsCfBxbsedm7hAHscUTiIyB9dL5p1gx2LMuSbow5WbosOrrrmTk3uSBI1XFD0p0EmIMcaYwLER/oxfRORuXJHzN6qaa9ufs0VVk7M0CDTGGFPEWNWMMcYYY4LGSkSMMcYYEzRFoo1I1apVNSYmJthhGGNMkbJ06dJ9qlot2HEYk5sikYjExMQQH59nr2FjjDE+RCS7UaiNKVSsasYYY4wxQWOJiDHGGGOCxhIRY4wxxgSNJSLGGGOMCRpLRIwxxhgTNJaIGGOMMSZoLBExxhhjTNBYImKMMYXQiRMwYQJMmRLsSIwJLEtEjDGmENm7F559FurUgTvvtETEnPssETHGmEJg7VoYMACio2HYMGjXDmbNgs8+C3ZkxgRWkRji3RhjzkWq8MMP8PLLMGMGlC4Nt94KDz0ETZsGOzpjzg5LRIwx5ixLTnZVLi+/DCtWQPXq8MwzcO+9UM2mqDPFjCUixhhzluzfD2PHwuuvw86d0KwZjB8PvXu70hBjiqOAJyIiEgrEAztUtbuI1AWmAFWApUBfVU0OdBzGGBMsGzbAq6/CxIlw/Dh07Qrvvuv+igQ7OmOC62w0Vn0QWOtzfyTwiqo2AA4Ad56FGIwx5qxShf/+F667Dpo0cSUfN98MP/8M334LV15pSYgxEOBERERqAVcD73j3BbgMmOqt8h5wfSBjMMaYsyklBSZPdr1eOnWChQvh73+HrVtdMtK8ebAjNKZwCXTVzKvAY0AF734V4KCqpnr3twNR2W0oIv2B/gDR0dEBDtMYY87MgQMwbhyMGgU7dkDjxq49SN++UKZMsKMzpvAKWImIiHQH9qjq0tPZXlXfVtV2qtqumjUjN8YUUhs3wgMPQO3a8Le/uQTkq69gzRro39+SEGPyEsgSkY7AtSLSDSgNhAOvAeeJSJhXKlIL2BHAGIwxpsCpuiqXl16Czz+HsDDX/mPwYGjVKtjRGVO0BKxERFWfUNVaqhoD3AT8oKp9gNnAjd5qtwFfBCoGY4wpSKmp8PHHcNFFEBcHc+bAE0/Ali3w3nuWhBhzOoIxjsjfgCki8hywHBgfhBiMMcZvhw7BO++49h9bt0KDBjBmDNx2G5QrF+zojCnazkoioqpzgDne/5uA9mfjuMYYcya2bHHJxzvvwJEjcOmlMHo0dO8OITZTlzEFwkZWNcaYLH780Q2//u9/u4SjVy8YMgTatg12ZMaceywRMcYYIC3NzXT78suwaBFUrAiPPAL33+96xBhjAsMSEWNMsXbkCEyYAK+9Bps3Q716rjqmXz8oXz7Y0Rlz7rNExBhTLG3b5hKOt9+Gw4ehY0f417/ckOyhocGOzpjiwxIRY0yxEh/vql8++cTdv/FGN/7HhRcGNy5jiitLRIwx57y0NPjyS5eAzJsHFSrAgw+6EVHr1Al2dMYUb5aIGGPOWUePwsSJ8Oqr8OuvLul4+WW4804IDw92dMYYsETEGHMO2rEDXn/dTTp34ICrdnn+eejRww3HbowpPOwtaYw5ZyxfDq+8Ah99BOnpLvEYMgQ6dAh2ZMaYnFgiYowp0tLTYcYMV+Uye7brcnvffa79R716wY7OGJMXS0SMMUXSsWMwaZIrAdmwAWrVghdfhLvugvPOC3Z0xhh/WSJijClSdu1yE869+Sbs3++GXf/wQ9cNt0SJYEdnjMkvS0SMMUXCypWu9OPDDyElxQ08NmQIxMWBSLCjM8acLktEjDGFlip8+y289BLMmgVly8Ldd8NDD0GDBsGOzhhTECwRMcYUOidOwAcfuBKQNWsgMhL++U/o3x8qVw52dMaYgmSJiDGm0NizB954w9327oVWreD996FXLyhZMtjRGWMCIWCJiIiUBuYCpbzjTFXVp0VkInApcMhb9XZV/SlQcRhjCr81a1zpx/vvQ1ISdO/u2n906mTtP4w51wWyRCQJuExVE0WkBDBfRL7xHntUVacG8NjGmEJO1bX7ePll+M9/oHRp6NfPtf9o3DjY0RljzpaAJSKqqkCid7eEd9NAHc8YUzQkJbmRT19+GX7+GWrUgGefhXvugapVgx2dMeZsCwnkzkUkVER+AvYA36nqYu+hf4jIShF5RURK5bBtfxGJF5H4vXv3BjJMY8xZsG8fPPecm3iuXz+37N134bffYOhQS0KMKa4CmoioapqqtgJqAe1FpDnwBNAEuACoDPwth23fVtV2qtquWrVqgQzTGBNA69a50o7ateHvf4c2beC772DFCrj9diiV7U8RY0xxEdBEJIOqHgRmA1ep6k51koB3gfZnIwZjzNmj6uZ9ueYaaNoUJk6EW26BVavcvDCXX26NUI0xTsASERGpJiLnef+XAa4A1olIhLdMgOuBVYGKwRhzdiUnu54vbdrAZZfB4sUwfDhs3QrjxkGzZsGO0BhT2ASy10wE8J6IhOISnk9U9SsR+UFEqgEC/ATcE8AYjDFnwe+/w9ixMHo07NwJsbHwzjvQp4/rDWOMMTkJZK+ZlUDrbJZfFqhjGmPOrl9+gddec41Ojx2DK66ACRPgyiut6sUY4x8bWdUYky+qMG+e6347fTqEhbmSjyFD4Pzzgx2dMaaosUTEGOOXlBSYOtUlIPHxUKUKPPUUDBwIERHBjs4YU1RZImKMydXBg66h6ahRsH07NGoEb74Jt97qZsM1xpgzYYmIMSZbmza59h/jx8PRo9C5s0tAunWDkLPS8d8YUxxYImKMyaQKixa56pfPPnMJx803w+DB0PqUpufGGHPmLBExxpCaCtOmuQRk8WI47zx47DG4/36Iigp2dMaYc5klIsYUY4cPu6qX115zc77Urw+vvw633Qblywc7OmNMcWCJiDHF0G+/ucan48bBkSNwySUuGeneHUJDgx2dMaY4sUTEmGJk8WJX/fLvf7v7vXq59h8XXBDcuIwxxZclIsac49LS4IsvXAKyYAFUrOgGHxs0yM2Ia4wxwZTvREREQoDyqno4APEYYwrIkSNu6PVXX4XNmyEmxv1/xx1QoUKwozPGGMev0QBE5EMRCReRcrjZcteIyKOBDc0Yczq2b3c9XmrXhgcfdKOeTp0Kv/7q7lsSYowpTPwdlijWKwG5HvgGqAv0DVhUxph8W7rUzflSty689BJ07erGBFmwAG64wRqhGmMKJ3+rZkqISAlcIvK6qqaIiAYwLmOMH9LT4csvXfuPuXNdacegQfDAA64qxhhjCjt/E5GxwBZgBTBXROoA1kbEmCA5ehTeew9eecVVuURHu1KQO+90jVGNMaao8CsRUdVRwCifRb+JSOfAhGSMyUlCghtw7K234MAB1+12yhRX9RJmfeCMMUWQXx9dIlIFeBqIAxSYD4wA9uexXWlgLlDKO9ZUVX1aROoCU4AqwFKgr6omn+6TMOZc99NPrvplyhQ3HHuPHq4LbocOIBLs6Iwx5vT521h1CrAXuAG40fv/Yz+2SwIuU9WWQCvgKhG5CBgJvKKqDYADwJ35DdyYc116Onz9NXTp4iacmzYN7r0XfvnFDUjWsaMlIcaYos/fRCRCVZ9V1c3e7TmgRl4bqZPo3S3h3RS4DJjqLX8P1wjWGAMcOwZjx0JsrBtyff16GDkStm1zw7DXrx/sCI0xpuD4m4jMFJGbRCTEu/UCvvVnQxEJFZGfgD3Ad8BG4KCqpnqrbAdOmd9TRPqLSLyIxO/du9fPMI0punbtgmHDXMPTe+6BcuVg8mQ3GNljj0GlSsGO0BhjCp6o5t0LV0SOAOWAdG9RCHDU+19VNdyPfZwHfAb8HZjoVcsgIrWBb1S1eU7btmvXTuPj4/OM05ii6OefXe+XyZMhJQWuuQYefthNRGdVL+ZMiMhSVW0X7DiMyY2/vWbOeCxGVT0oIrOBi4HzRCTMKxWpBew40/0bU5SowsyZrsvtd99BmTJw111u5NNGjYIdnTHGnD1+d/gTkWuBP3l356jqV35sUw1I8ZKQMsAVuIaqs3GNXqcAtwFf5DdwY4qiEydcycfLL8OaNW749eefh/79oUqVYEdnjDFnn7/dd18ALgAme4seFJGOqvpEHptGAO+JSCiuOucTVf1KRNYAU0TkOWA5MP70wjemaNizB958E8aMgb17oWVLNyDZTTdByZLBjs4YY4LH3xKRbkArVU0HEJH3cAlEromIqq4EWmezfBPQPn+hGlP0rFnjZrydNAmSkuDqq934H507W/sPY4yBfFTNAOcBv3v/2yDSxuRAFb7/3lW/fPMNlC4Nt90GgwdDkybBjs4YYwoXfxORfwLLvcamgmsr8njAojKmCEpKgo8+cgnIzz9D9eowYoTrilutWrCjM8aYwsnfXjMficgcXDsRgL+p6q6ARWVMEbJ/v5v75fXX3VggzZvDhAlw882uNMQYY0zO/G2sKkAXoJ6qjhCRaBFpr6pLAhueMYXX+vWu/cd778Hx43Dlle7/K66w9h/GGOMvf6tm3sANZnYZbrK7I8C/+aOExJhiQRX++183/sdXX7keL337wkMPuZIQY4wx+eNvInKhqrYRkeUAqnpARKzToSk2kpPhk09c+4/ly6FqVXj6aTcJXY08Z10yxhiTE38TkRRvLBCFzIHK0nPfxJii78ABNwHd6NGQkOB6vbz9NtxyixsN1RhjzJnxNxEZhZsnprqI/AM3KurfAxaVMUH2669uptsJE9xsuF26wLhxcNVVEOLvVJHGGGPy5G+vmckishTXYFWA61V1bUAjM+YsU4X58131yxdfQFgY9O7txv9o2TLY0RljzLnJ314zd6rqeGCdz7IXVNXGEjFFXkoK/PvfrgFqfDxUrgxPPgn33efmgjHGGBM4/lbN3CAiJ1R1MoCIjAFshARTpB08CO+8A6NGwbZt0LAhvPEG3HorlCsX7OiMMaZ48DsRAaaLSDpwFXBQVe8MXFjGBM7mza79x/jxkJgInTq5yeiuvtrafxhjzNmWayIiIpV97t4FfA4sAJ4Rkcqq+nv2WxpT+Cxa5Np/TJvmEo6//tW1/2jbNtiRGWNM8ZVXichSXJdd8fl7tXdToF5AozPmDKWmwmefuQTkxx/hvPPg0Ufh/vuhVq1gR2eMMSbXRERV656tQIwpSIcPu6qX116D336DevXcWCC33w7lywc7OmOMMRn8bSNiTJGwdatrfDpunEtG4uLglVfg2mshNDTY0RljjMkqYE3zRKS2iMwWkTUislpEHvSWDxeRHSLyk3frFqgYTPGxZAncdJMr+Xj1VejWDRYvhnnzoEcPS0KMMaawyquxaglVTTnNfacCD6vqMhGpACwVke+8x15R1X+d5n6NASAtDaZPd+0/5s+H8HA3+dwDD0B0dLCjM8YY44+8qmYWich24D/Af1R1i787VtWdwE7v/yMishaIOt1AjcmQmAjvvutKPjZtgjp1XPXLHXe4ZMQYY0zRkWvVjKq2Ax7y7r4qIv8TkVdEpKuIlPL3ICISA7QGFnuL7heRlSIyQUQq5bBNfxGJF5H4vXv3+nsocw7bvh3+9jeoXduVelSvDp9+6uaFeeghS0KMMaYoElX1f2WREsAluEHNOgF7VfXqPLYpD/wX+IeqThORGsA+XPffZ4EIVb0jt320a9dO4+Pj/Y7TnFt273YJyOTJkJ4Of/kLDBkCF18c7MiMKdxEZKn3g9KYQitfvWa89iI/eDdEJNeqFi9x+TcwWVWnefvY7fP4OOCrfMZsiglVeP99V9px7Jib++XBB6GudSo3xphzxhl131XVHTk9JiICjAfWqurLPssjvPYjAD2AVWcSgzk3bd0KAwbAf/4DHTq4MUGaNAl2VMYYYwpaIMcR6Qj0BX4WkZ+8ZU8CN4tIK1zVzBZgQABjMEVMejq89ZarilF1Y4IMHGjdb40x5lwVsEREVefjhoTPakagjmmKtl9+gbvugrlz4fLL3aBkMTHBjsoYY0wg+ZWIiEgj4FGgju82qnpZgOIyxUhqqut+O2wYlCrlqmH69QPJLo01xhhzTvG3RORT4C1gHJAWuHBMcbNyJdx5J8THw/XXw5gxEBkZ7KiMMcacLf4mIqmq+mZAIzHFSlISPP+8u1WqBB9/DD17WqXp5KcAACAASURBVCmIMcYUN3kN8V7Z+/dLERkIfAYkZTyuqr8HMDZzjlq82JWCrF4Nt9ziRkitUiXYURljjAmGvEpEluJ6t2T8Tn3U5zEF6gUiKHNuOnYM/v53l3hERsLXX7vJ6YwxxhRfuSYiqmpDR5kCMXu26xGzaRPccw+MHGlDshtjjMljrpkMInKfiJznc7+SV1VjTK4OHXIDk112mWv/MXs2vPmmJSHGGGMcvxIR4G5VPZhxR1UPAHcHJiRzrvjqK2jWDN55Bx55xPWQ6dQp2FEZY4wpTPxNREK9IdsBEJFQoGRgQjJF3b590KcPXHON6xGzaBG8+CKULRvsyIwxxhQ2/iYi3wIfi0gXEekCfAT8J3BhmaJIFaZMgaZN4dNP4emnYelSaN8+2JEZY4wprPwdR+QxoD9wr3f/O+CdgERkiqSEBLj3Xpg+HS64wI2Oev75wY7KGGNMYZdnIuJVw0xS1T640VWNyaQKEybAww+7Qcr+9S948EEIC+R0isYYY84ZeX5dqGqaiNQRkZKqmnw2gjJFw6ZN0L8/fP89XHqpa5TaoEGwozLGGFOU+Pu7dROwQESmA0czFqrqywGJyhRqaWkwejQ89RSEhsJbb8Hdd0OIvy2OjDHGGI+/ichG7xYCVAhcOKawW7vWDc++aJEbFfWtt6B27WBHZYwxpqjyKxFR1WcARKS8dz8xr21EpDYwCaiBGw7+bVV9zZu/5mMgBtgC9PLGJTGFWEoK/N//wYgRUL48vP++66Jrk9QZY4w5E/6OrNpcRJYDq4HVIrJURJrlsVkq8LCqxgIXAfeJSCzwOPC9qjYEvvfum0Js2TLXE2boULj+elcqcsstloQYY4w5c/7W6r8NDFHVOqpaB3gYGJfbBqq6U1WXef8fAdYCUcB1wHveau8B159O4CbwTpyAJ55w44Ds3g2ffQYffwzVqwc7MmOMMecKf9uIlFPV2Rl3VHWOiJTz9yAiEgO0BhYDNVR1p/fQLlzVjSlk5s93bUE2bIB+/eCll9woqcYYY0xB8rdEZJOI/F1EYrzbUFxPmjx57Ur+DTykqod9H1NVxbUfyW67/iISLyLxe/fu9TNMc6YSE2HQIPjTn9y4IDNnunFCLAkxxhgTCP4mIncA1YBpuKSiKtAvr41EpIS3/mRVneYt3i0iEd7jEcCe7LZV1bdVtZ2qtqtWrZqfYZozMXMmNG8OY8a4ZGTVKrjiimBHZYwx5lzmbyJyuao+oKptVLWtqj4E5PoV5U2SNx5Ym2W8kenAbd7/twFf5DdoU7AOHHDVL1deCaVLw7x58NprrneMMcYYE0j+JiJP+LnMV0egL3CZiPzk3boBLwBXiMgvwOXefRMk06ZBbKzrjvvkk/DTT9CxY7CjMsYYU1zk2lhVRP4MdAOiRGSUz0PhuO65OVLV+UBOHTy75CdIU/B274b774epU6FVK5gxA1q3DnZUxhhjipu8es0kAPHAtcBSn+VHgMGBCsoEjip88AE89JBrmPqPf8Cjj0KJEsGOzBhjTHGUayKiqiuAFSJyAPhKVdPPTlgmELZuhXvugW++gQ4dYPx4aNIk2FEZY4wpzvxtI9IL+EVE/k9E7KuriElPhzffhGbN4L//dQ1R5861JMQYY0zw+ZWIqOotuAHJNgITRWSRN86HTYBXyP3yC3TuDAMHwkUXuS65DzzgZs01xhhjgs3vidu9wcimAlOACKAHsExEBgUoNnMGUlPhxRehRQtYscJVw8ycCXXrBjsyY4wx5g9+DfEuItfiBjBrgJtRt72q7hGRssAaYHTgQjT5tXKlG549Ph6uuw7eeAMiI4MdlTHGGHMqf+eauQF4RVXn+i5U1WMicmfBh2VOR1ISPP+8u1Wq5Cao69nTZsk1xhhTePmViKjqbSJS0ysZUeB/qrrLe+z7QAZo/LN4sSsFWb0abrkFXnkFqlYNdlTGGGNM7vxqI+KVeiwB/gLcCPwoIncEMjDjn2PH4OGHXXfcQ4fgq6/cKKmWhBhjjCkK/K2aeQxorar7AUSkCrAQmBCowEze5syBu+6CjRthwAD4v/+D8PBgR2WMMcb4z99eM/txo6lmOOItM0Fw6JBLPDp3dvdnz4a33rIkxBhjTNHjb4nIr8BiEfkC10bkOmCliAwByDK7rgmgr792ScjOna5KZsQIKFs22FEZYwIhXdM5kXqCsiXsTW7OXf4mIhu9W4YvvL82oNlZsm+fmx9m8mQ3Quq0adC+fbCjMsacrsNJh0k4ksCOwztIOJLg/j+y46S/O4/s5NrG1zK119Rgh2tMwPjba+aZQAdisqfquuEOGuSqZJ5+Gp58EkqWDHZkxpjsJKUmsTNxZ2aCkZFUZP0/MTnxlG3DS4UTVSGKyAqRdIrpRGT5SNpGtg3CszDm7PG3RMQEQUIC3HsvTJ8OF1zgRkc9//xgR2VM8ZSWnsaeo3tOLb04vIOExITM//cfP7X5XKnQUkRWiCSyQiStaraiW4NuRFaIJCrcJR1RFaKIqBBB+ZLlg/DMjAkuS0QKIVWYMMG1AUlKckO1P/QQhNmrZUyBU1UOnjiYZ4KxK3EXaZp20rYhEkKNcjWIrBBJzHkxdKjVITO5yEgwIitEUrlMZcRGFjQmWwH9ahORCUB3YI+qNveWDQfuBvZ6qz2pqjMCGUdRsnkz3H03fP89/OlP8M470LBhsKMypmg6nnL81OoRL8HwbZtxPPX4KdtWKl0pM6mIrRabmVRk/I2sEEmN8jUIC7FfCMacCX/nmmkEvAnUUNXmItICuFZVn8tj04nA67j5aXy9oqr/ym+w57K0NHj9ddf+IzQU3nwT+veHEL+nJTSm+EhNT2V34u6TSy+OJJySYBw4ceCUbcuElclMMC6IuiDbBCOyQiRlSpQJwjMzpvjxN5UfBzwKjAVQ1ZUi8iGQayKiqnNFJOZMAiwO1q51w7MvWgR//jOMHQu1awc7KmPOPlVl//H9J5deZNOjZHfibhQ9adtQCSWiQgSRFSJpVKWRa+zpk2BkJB8VS1W0ahJjChF/E5Gyqroky5s39QyOe7+I3ArEAw+r6ik/W0SkP9AfIDo6+gwOVXilpLjRUEeMgPLl3dDsffrYJHXm3JSYnHhKgpFdj5LktORTtq1atmpmUtGqZqtsE4xqZasRGhIahGdmjDkT/iYi+0SkPm4wM0TkRmDnaR7zTeBZb1/PAi8Bp8xbo6pvA28DtGvXTrM+XtQtW+ZKQX76yc2QO3o01KgR7KiMyb/ktGR2Je7KtvTC9//DSYdP2bZciXJEhUcRVSGKjrU7nlQ9kpFgRJSPoFRYqSA8M2PM2eBvInIfLiloIiI7gM3ALadzQFXdnfG/iIwDvjqd/RRVJ07AM8+4njDVqrmByXr0CHZUxpwqXdPZd2xfrgNuJRxJYM/RPadsWyKkRGZC0axaM7rW63pScpFRolGhlI2JaExx5++AZpuAy0WkHBCiqkfy2iYnIhKhqhmlKT2AVae7r6Jm/nxXCrJhA/TrBy+9BJUqBTsqU9yoauaonjmVXuw4vIOdiTtJTT+5BlYQqpernplItI9sn22CUaVsFULEWlobY/Lmb6+Z84BbgRggLKOtiKo+kMd2HwGdgKoish14GugkIq1wVTNbgAGnF3rRkZgITzwBY8ZAdDTMnAlXXBHsqMy56ETqCXYe2ZnreBgJRxI4mnL0lG0rlqqYmVR0rtuZyPInD7gVWSGSmuVrUiK0RBCemTHmXOVv1cwM4EfgZyDd352r6s3ZLB7v7/bngpkzXTfcrVvh/vvh+eddw1Rj8sN3VM/cuqzmNqpnVHgUrSNa071R91MG3IqsEEm5kuWC8MyMMcWdv4lIaVUdEtBIzjEHDsCQITBxIjRuDPPmQceOwY7KFDYZo3qeMuBWliqT3Eb1jAqPom6lusRFx52SYESFR1GpdCXrrmqCaunSpdXDwsLeAZoDVmdXvKQDq1JTU+9q27btqQ3K8D8ReV9E7sY1LE3KWKiqv595jOeezz6DgQNh715XJTNsGJQuHeyozNl2LOVYnuNhJBxJ4ETqiVO2rVymcmZS0bx681NKL6LCo6herrqN6mmKhLCwsHdq1qzZtFq1agdCQkLOuV6QJmfp6emyd+/e2F27dr0DXJvdOv5+iiUDLwJPQeYoQgrUO+MozyG7d7tZcj/9FFq1gq+/hjZtgh2VKWip6ansStyVZ4Jx8MTBU7bNGNUzqkIUF0ZdmG2CEVE+wkb1NOea5paEFE8hISFarVq1Q7t27Wqe0zr+JiIPAw1UdV/BhHZuUYUPPnAT0yUmwj/+AY8+CiWsTV+R4juqZ24DbuU2qmdUhSgaV2lM55jOpwy4ZaN6mmIsxJKQ4st77XOskvM3EfkVOFYgEZ1jtm6Fe+6Bb76Biy+G8eOhadNgR2WySkxOzHM8jNxG9cxIKtrUbHPKgFtRFaKoWraqjeppjDGnwd9E5Cjwk4jM5uQ2Irl23z2Xpae7OWEee8z9/9prcN99bsI6c/YkpyVndlfNKcHYcXgHR5JPHfqmfMnymQlGXHQckeVPTTBqlq9po3oaUww9/vjjNV944YVdBbGvIUOGRJYvXz5txIgRu7N7vHfv3tG33377/qFDh9b617/+te1Pf/pTofvh/+abb1Z+5ZVXagKUK1cu/Y033vjt4osvPnXa6tPgbyLyuXczwC+/wF13wdy5cPnl8PbbULdusKM6N6kqG/ZvYNH2RWw5uOWUKdz3Htt7yja+o3o2r96crvW6njLgVmSFSBvV0xiTo1GjRkXkJxFJT09HVQk9jV+jy5YtKz9p0qSt+d7wLGrQoEHSggUL1lerVi3tk08+CR8wYECdlStXriuIffs7sup7BXGwoi41FV55xfWCKVXKVcP062eT1BWklLQUlu1cxvyt85m/bT4Lti7ITDYyRvWMCo+idsXaXBh1YbYJho3qaYzxtX79+pJXXXVVw/PPP//YqlWryjZq1Oj4p59+umX27NnlRo0aVX3WrFkbAT777LPwN954o1rDhg1PJCUlhTRp0iS2UaNGx6dPn755+PDhNSZPnlwVoG/fvnuHDRu2Z/369SWvvPLKRq1bt078+eefy82YMeOXlStXlh42bFhUWlqaVK5cOXXRokUbANauXVumffv2jRMSEkrec889u4cOHboHYNmyZaXr1at3Iizsj6/jtLQ0evXqFRMVFZU8atSohGnTpoWPGDEiMjk5WerUqZM0ZcqULRUrVkz/4osvKjz++OO109LSaNmy5bFJkyb9VqZMGY2Kijr/mmuuOfDDDz+ElypVSj/66KNNzZs3T5owYUKlf/7zn5EhISFaoUKFtPj4+PX+nsMrrrgicxTEzp07H73//vtLFtDLk3siIiKfqGovEfkZOKWhkaq2KKhACruff4Y77oD4eLjuOnjjDYiMDHZURd/hpMP8uP1Hl3hsnc+P23/keKor7atfqT7dGnYjLjqOjrU70qByAxvV05gi7o47qL1qFWULcp/Nm3NswgS25bbOli1bSo8dO3ZL165dj/bs2TPmxRdfrDZ8+PDdDz74YHRCQkJYZGRk6oQJE6r069dvX+/evQ9NnDix+rp169YAzJs3r+yHH35YZenSpWtVlbZt2zbt0qXLkapVq6Zt3bq11Pjx4zd36dJlS0JCQtj9998fM2fOnHVNmjRJ3r17d2bxyK+//lp64cKF6w8ePBjatGnT5o8++ujeUqVK6fTp0yt27dr1UMZ6KSkpcv3119eNjY09PnLkyF07d+4Me/755yPmzp27ITw8PP2pp56q+eyzz9YYMWLErgEDBtSdOXPm+hYtWiT16NEj5sUXX6w2bNiwPQAVK1ZM3bBhw5rXX3+9yqBBg2rPnj371xdeeCFi5syZG+rWrZuyb9++025IMHr06KqdO3c+lPea/smrRORB72/3gjpgUZOc7HrBPP+8mxdmyhTo1ctKQU5XwpGEzKRj/tb5rNi9gnRNJ0RCaF2zNf3b9s9MPCIqRAQ7XGPMOaJmzZrJXbt2PQrQt2/f/aNGjaoeEhKyu1evXvvHjRtX+b777tu/bNmy8tOmTducdds5c+aU79at28Hw8PB0gKuvvvrA7NmzK/Ts2fNgREREcpcuXY5665Vr3779kSZNmiQD1KhRI3MUwq5dux4sU6aMlilTJrVy5cop27dvD6tfv37KrFmzwj/44IMtGesNHDiwzvXXX//7yJEjd2Xsc+PGjaXbt2/fBFyi0rZt28QVK1aUrlWrVlKLFi2SAG6//fb9Y8aMqQ7sAbjtttt+B7j77rt/Hzp0aG2Adu3aJfbp0yfmhhtuONCnT58Dp3Mev/zyywoffPBB1YULFxZItQzkkYj4TE43UFX/5vuYiIwE/nbqVueOJUtcKcjq1dCnD7z6KlStGuyoig5VZd2+dczfOp95W+cxf+t8Nh907/GyJcpyUa2LGHrJUOKi47io1kXWZsOYYiCvkotAydptPuP+vffeu//qq69uULp0ab3mmmsOlMjnuAtly5b1a9qTUqVKZdYqhIaGkpqaKkeOHAk5fPhwaExMTErGY+3atUucN29e+LFjx3aXLVtWVZW4uLjDX3755UkJ0qJFi3IdbCgk5I/qaRFRgA8//HDrDz/8UG769OkV27ZtG7t06dI1NWvWzEyWBg0aFPXdd99VBMgoDfK1ePHiMgMHDqzz9ddf/+K73ZnytyI9uyna/lxQQRQ2x47BI4+47rgHD8JXX7lxQiwJyV1yWjKLti3ixQUvct2U66j2YjVi34il/1f9+Xbjt7SOaM3LXV9myV1LOPi3g3x/6/c80/kZrqh/hSUhxpiA2rlzZ8lZs2aVA5g8eXLlDh06JALExMSk1KhRI+Wll16K6N+/f+ZYWWFhYZqUlCQAnTt3TpwxY8Z5XuIQMmPGjEqdO3c+pStep06dji5ZsqTCunXrSgL4Vs1k5+uvv64QFxd30n4GDBiwr2vXroe6d+9ePyUlhU6dOh2Nj48vv2rVqlIAhw8fDlm5cmWpli1bntixY0fJjOWTJk2qcskll2Tua9KkSZUBxo8fX6l169ZHAVavXl3qsssuO/rqq68mVKpUKXXTpk0ntfMYPXr0jnXr1q3JLgn55ZdfSvbs2bP+hAkTNmeUwhSUvNqI3AsMBOqJyEqfhyoACwoykMJizhzXI2bjRhgwAEaOhIoVgx1V4XTwxEEWbVuU2bB0yY4lmcOVN6zckOsaX0dcdBxx0XE0qNzABvIyxgRNTEzMidGjR1fv379/2YYNG5545JFHMrvc3XTTTfvHjBkT1qZNm8z5Fvr06bO3adOmsc2bNz82ffr0zb17997fpk2bpuAaq3bs2PH4+vXrT/oij4yMTB01atSWHj16NEhPT6dKlSopCxcu/CWnmGbMmFGxV69ep1SRDB8+fPfgwYND//KXv9T9/PPPN48dO3bLTTfdVC85OVkAnn766R0tWrRIeuutt7b07NmzfkZjVd/ndODAgdBGjRrFlixZUqdMmbIJYPDgwbW2bNlSSlUlLi7u8EUXXeR399uhQ4dGHDx4MGzQoEF1wCVqq1atWuvv9rkR1ZwHuxORikAl4J/A4z4PHTmb88y0a9dO4+PjA3qMQ4fcmCBvvw3168O4cdC5c0APWeRsP7ydeb/Ny0w8ft79M4oSKqG0iWiTmXR0rN2RGuVrBDtcY4o9EVmqqu2CHceKFSu2tGzZMmgjc69fv75k9+7dG/7yyy+rs3v81ltvjW7duvWxwYMHn9UYY2Njmy5fvnydb7VNQYiKijo/Pj5+bURERGpB7vdMrFixomrLli1jsnssrzYih4BDwM0BiKvQ+PprV/qxcyc8/DCMGAFlC7RNd9GTrums2bvmpIalvx36DXADgV1c62Ju6HQDcdFxXBh1oU0hb4wpkpo1a9a0TJky6WPHjj3rbVfWrFlTICUKRV2xnrpz3z43P8zkydCsGUybBu3bBzuq4DiReoL4hPjMpGPBtgWZk7bVLF+TS6IvYcjFQ4iLjqNFjRY266sxpsho3Lhxck6lIatXrz7nkoEdO3b8HOwY8iOg3yYiMgHX9XePqjb3llUGPgZigC1AL1U9rW5Ep0sVPvnEzZR74AA8/TQ8+SSULLDhWQq/A8cPsHDbwsweLf9L+F/mPCtNqjbhxqY3EhcdxyV1LqHueXWtfYcxxpiACPTP2onA68Akn2WPA9+r6gsi8rh3/6x1A05IgIED4YsvoF07+P57OP/8s3X04FBVth7a+kc1y7b5rNqzCoCwkDDaRbbjgfYPEBcdR4faHahWrlqQIzbGGFNcBDQRUdW5IhKTZfF1QCfv//eAOZyFREQVJkxwbUCSkuDFF121TNg5WMOQlp7Gqj2rMpOO+Vvns/3wdgDCS4XToXYHbmp2E3HRcVwQdQFlSxTzBjHGGGOCJhhfwzV8BkrbBWTbvUJE+gP9AaKjo8/ogJs3Q//+MGsW/OlP8M470LDhGe2yUDmecpz/JfzP9WjZNp+F2xZyOOkwAJEVIrkk+hJXzRJ9Cc2rN7fp6o0xxhQaQZ0ZTF3f4Wy7Lanq26raTlXbVat2elUFaWnw2mvQvDn8+KObH2b27KKfhOw7to/p66fz2HeP0WF8Byq+UJFLJ17K0NlD2XZoGzc3v5n3e7zP5gc3s33wdqbcOIX7299Py5otLQkxxhg/PP744zULal9DhgyJHDZsWI5jGvTu3Tt65syZ5dq3b9947ty5hbKIevny5aVbtWrVpGTJkm2yPpepU6eGx8TENI+Ojm7+5JNP5vu8BaNEZLeIRKjqThGJwBsXPxCWLHHVL1ddBWPHwhkWrASFqrL54OaTutGu3ecaeZcMLckFkRdk9mbpULsDlctUDnLExhhT9I0aNSrihRde2OXv+unp6agqoaH5/7G3bNmy8pMmTdqa7w3PourVq6e+9tprW6dOnVrJd3lqaiqDBw+O/vbbbzfUq1cvpWXLlk1vuOGGg23btj2R076yCkaJyHTgNu//24AvAnWgiy+GBQtgxoyik4SkpaexfOdyRi0eRa9PexH1chT1R9Xnts9v45PVn1C3Ul2ev+x55t4+l0OPH2L+HfN54fIX6N6ouyUhxhiTjfXr15esW7dus2uvvbZuvXr1ml111VX1jhw5EjJ9+vQKl19+ef2M9T777LPwK664ov7AgQOjkpKSQpo0aRJ77bXX1gUYPnx4jYYNGzZr2LBhsxEjRlTP2G9MTEzzHj16xDRq1KjZxo0bS06dOjU8Nja2aePGjWMvvvjiRhn7Xrt2bZn27ds3rlWr1vnPPfdc9Yzly5YtK12vXr0TYT4NFtPS0rjhhhtiHnjggUiAadOmhbdq1apJbGxs0z//+c/1Dh06FALwxRdfVGjatGlso0aNYnv27Blz/PhxATeg2T333FOrUaNGseeff37TjGHgJ0yYUKlhw4bNGjduHNuuXbvG+TmHUVFRqZdeeumxEiVKnFSLMWfOnHJ16tRJio2NTS5durT+5S9/+X3q1Knn5Wffge6++xGuYWpVEdkOPA28AHwiIncCvwG9AhlDhw6B3PuZO5p8lCU7lmQ2LF20bRFHkt10AdEVo+lctzNxtd2Ipc2qNyNEglqbZowxZ+SOL+6ovWrPqgKtfmhevfmxCddNyHVAsi1btpQeO3bslq5dux7t2bNnzIsvvlht+PDhux988MHohISEsMjIyNQJEyZU6dev377evXsfmjhxYvWMOVfmzZtX9sMPP6yydOnStapK27Ztm3bp0uVI1apV07Zu3Vpq/Pjxm7t06bIlISEh7P7774+ZM2fOuiZNmiT7zjXz66+/ll64cOH6gwcPhjZt2rT5o48+urdUqVI6ffr0il27dj2UsV5KSopcf/31dWNjY4+PHDly186dO8Oef/75iLlz524IDw9Pf+qpp2o+++yzNUaMGLFrwIABdWfOnLm+RYsWST169Ih58cUXqw0bNmwPQMWKFVM3bNiw5vXXX68yaNCg2rNnz/71hRdeiJg5c+aGunXrpuzbt69A6um3bdtWMioqKjnjfq1atZIXL15cPj/7CHSvmZxGZO0SyOMWZnuO7mHB1gWZiceynctITU9FEM6vcT59W/R1w6RHdyS6YhEpxjHGmEKuZs2ayV27dj0K0Ldv3/2jRo2qHhISsrtXr177x40bV/m+++7bv2zZsvLTpk3bnHXbOXPmlO/WrdvB8PDwdICrr776wOzZsyv07NnzYERERHKXLl2OeuuVa9++/ZEmTZokA9SoUSNzhtquXbseLFOmjJYpUya1cuXKKdu3bw+rX79+yqxZs8I/+OCDLRnrDRw4sM7111//+8iRI3dl7HPjxo2l27dv3wRcotK2bdvEFStWlK5Vq1ZSxgR0t99++/4xY8ZUx2vucNttt/0OcPfdd/8+dOjQ2uBm9u3Tp0/MDTfccKBPnz5ndfyu3JyDnVcLD1Vl44GNJ83PsmH/BgBKhZaifVR7Hu3waGb7jvNK56s0yxhjipy8Si4CJeugjBn377333v1XX311g9KlS+s111xzoESJEvnab9myZdP9Wc93PpnQ0FBSU1PFm803NCYmJiXjsXbt2iXOmzcv/NixY7vLli2rqkpcXNzhL7/88qQEadGiRWVyO15IyB+l5yKiAB9++OHWH374odz06dMrtm3bNnbp0qVratasmZksDRo0KOq7776rCJDdDLzZqV27dvKOHTsyhwPdvn37SSUk/rBy/gKUmp5KfEI8r/74Kjd+ciMRL0XQcHRD7ph+B5+t+4zGVRoz8vKRLLhjAYceP8TcfnN5vsvzdGvYzZIQY4wJoJ07d5acNWtWOYDJkydX7tChQyJATExMSo0aNVJeeumliP79+2dOehcWFqZJSUkC0Llz58QZM2ac5yUOITNmzKjUuXPnI1mP0alTp6NLliypsG7dupIAvlUz2fn6668rxMXFnbSfAQMG7OvateuhfodQCwAAF6lJREFU7t27109JSaFTp05H4+Pjy2e08zh8+HDIypUrS7Vs2fLEjh07SmYsnzRpUpVLLrkkc1+TJk2qDDB+/PhKrVu3PgqwevXqUpdddtnRV199NaFSpUqpmzZtOmk88dGjR+9Yt27dGn+TEIBLL7306JYtW0qvW7eu5IkTJ2TatGmVb7jhhoP+bg9WInJGEpMT+XH7j5m9WX7c/iNHU44CUPe8unSt3zVzRtomVZtY+w5jjAmSmJiYE6NHj67ev3//sg0bNjzxyCOP7M147Kabbto/ZsyYsDZt2mT29OjTp8/epk2bxjZv3vzY9OnTN/fu3Xt/mzZtmgL07dt3b8eOHY+vX7/+pC/yyMjI1FGjRm3p0aNHg/T0dKpUqZKycOHCX3KKacaMGRV79ep1ShXJ8OHDdw8ePDj0L3/5S93PP/9889ixY7fcdNNN9ZKTkwXg6aef3tGiRYukt956a0vPnj3rp6Wl0bJly2O+z+nAgQOhjRo1ii1ZsqROmTJlE8DgwYNrbdmypZSqSlxc3OGLLrrouL/nb+vWrWEXXHBB7NGjR0NFRMeOHVtj7dq1qypXrpz+0ksvbb3qqqsapaWl0bt3733t2rXzu8cMgLihPAq3du3aaXx8fLDDYFfirpO60f606yfSNA1BaFmzZWaj0o7RHakVXivY4RpjijkRWaqq7YIdx4oVK7a0bNlyX95rBsb69etLdu/evWFOE9/deuut0a1btz42ePDgsxpjbGxs0+XLl6/zrbYpCFFRUefHx8evjYiISC3I/Z6JFStWVG3ZsmVMdo9ZiUgOVJUN+zecNEz6r7//CkCZsDJcWOtCnoh7grjoOC6qdREVS1cMcsTGGGPyq1mzZk3LlCmTPnbs2LPedmXNmjXn3My/p8MSEU9yWjLLdy4/KfHYd8wlx1XLViUuOo572t5DXHQcrSNaUzK0GE3Va4wxRVjjxo2TcyoNWb169TmXDOzYsePnYMeQH8U2ETmcdJgft/+YOT/L4u2LOZ7qqsvqV6rP1Q2vzpyfpVGVRqe0uDbGGGPMmSs2iUjCkYST2nes2L2CdE0nREL+v717j66iuvcA/v0lIQkJSSABQiCEEzSvAxIgaQoYLS9TEKQCDUW4ivZaVJTaULz1cUspbb1axGVR1xUt1MutlF4RNdK0AjUssFhsAkTDI8gjRgJIQF4GyOv87h+zTzjEBDjkMXl8P2vNypw9c/b8zszA/M6efWZjSK8hmJ0y2+rf0fdmRIVE2R0uERFRh9CuE5GDpw5i4aaF+LDkQxw6bf0EO6hTEIZHD8fPb/050mPS8e0+30ZIQIjNkRIREXVM7ToRCfANwPsH3kd6TDrmps1Fekw6BvcajE6+3j2whoiIiJpHu36wRZ/QPjj202N4a9pbyBqehW/1+RaTECIiuqrHH3/c6+HsGzJv3rzeCxYsiGxo+YwZM2LWr18fnJaWlrB58+YmHYenqaxbty4kJCRkcGJiojMxMdE5f/782j4Ma9asCXU4HANjYmIGPvnkk17vt3adiADffKwvERHR1SxdutSrzoIulws1NTVXX7Ee27dv7zJ69Ojy63pzC0pNTf3a/eTV55577igAVFdXIysrKyYnJ2ffvn37dr311lvh+fn5gd7U2+4TESIi6tiKior8Y2NjB0yaNCm2f//+A8aNG9f/3LlzPtnZ2SFjx469wb3e22+/HXrbbbfdMGfOnD4VFRU+iYmJzkmTJsUCwMKFCyPj4uIGxMXFDVi0aFFPd70Oh2Pg5MmTHfHx8QMOHDjgv2bNmlCn05mUkJDgHD58eLy77j179nROS0tLiI6OvunXv/51T3f59u3bA/v373/Rz+9ST4mamhpMnTrV8eMf/7g3AKxduzZ08ODBiU6nM2n8+PH9z5w54wMA7777bkhSUpIzPj7emZmZ6bhw4YIA1gPNHnzwwej4+HjnTTfdlOR+DPyKFSu6xcXFDUhISHCmpqYmNMW+3bRpU3C/fv0qnE5nZWBgoE6ZMuWrNWvWeDVmSbvuI0JERK3MD3/YF4WFTXv7YeDA81hx5cH0iouLA5ctW1ackZFRnpmZ6Vi8eHGPhQsXfvnoo4/GHDlyxK93797VK1asiLjvvvtOzJgx48zrr7/e0z3mypYtW4JWrVoVkZ+fv0dVkZKSkjRmzJhz3bt3rykpKQlYvnz5oTFjxhQfOXLE75FHHnFs2rRpb2JiYqXnWDP79+8P3Lp1a9Hp06d9k5KSBj722GNlAQEBmp2dHZaRkXHGvV5VVZXceeedsU6n88Kzzz577OjRo35PP/101ObNm/eFhoa6nnrqqV6/+tWvIhctWnTsgQceiF2/fn3RoEGDKiZPnuxYvHhxjwULFhwHgLCwsOp9+/btfumllyLmzp3bNzc3d/8zzzwTtX79+n2xsbFVJ06cuOI4OPXZsWNHl4SEBGdkZGTV888//0VqaurFL7744rJB7qKjoyu3bdvWxZt62SJCRETtXq9evSozMjLKAeDuu+8+uXXr1i4+Pj6YNm3ayddeey38xIkTvtu3b++SmZl5pu57N23a1OX2228/HRoa6goLC3NNmDDhVG5ubggAREVFVY4ZM6bcrBeclpZ2LjExsRIAIiMja+/VZGRknO7cubNGRUVVh4eHVx0+fNgPADZu3Bh65513nnWvN2fOnH7uJMRd54EDBwLT0tISExMTnatXr44oKSnxLygoCIyOjq4YNGhQBQDce++9Jz/88MPan4DOmjXrKwD40Y9+9NWOHTu6ANatlZkzZzqWLFnSvbrau6e/jxgxovzzzz//pKioaPfDDz98fOrUqTd6VcEVsEWEiIhazlVaLppL3f6C7tcPPfTQyQkTJtwYGBiod9xxx6lOnbz7QUNQUJDrWtbzHE/G19cX1dXVYkbz9XU4HFXuZampqV9v2bIl9Pz5818GBQWpqiI9Pf3se++9d8izvo8++qjzlbbn43OpnUFEFABWrVpV8sEHHwRnZ2eHpaSkOPPz83f36tWrNlmaO3dunw0bNoQBQN0ReMPDw2s/5w9+8IMz8+bNizl69Khf3759K0tLS2sfNX748OHLWkiuhW0tIiJSLCKfishOEbF/RDsiImq3jh496r9x48ZgAHjjjTfCR4wY8TUAOByOqsjIyKolS5ZEzZ49u3bQOz8/P62oqBAAGDVq1Nc5OTldTeLgk5OT023UqFHn6m5j5MiR5R9//HHI3r17/QHA89ZMff7yl7+EpKenX1bPAw88cCIjI+PMxIkTb6iqqsLIkSPL8/Lyurj7eZw9e9bnk08+CUhOTr5YWlrq7y5fuXJlxC233FJb18qVK8MBYPny5d2GDBlSDgC7du0KGD16dPkLL7xwpFu3btUHDx68bKySF198sdTdGbVurCUlJX4ul5WL5ObmBrlcLkRGRlZ/5zvfKS8uLg7cu3ev/8WLF2Xt2rXhU6dOPX2lz12X3S0io1TVthEZiYioY3A4HBdffPHFnrNnzw6Ki4u7OH/+/DL3sunTp598+eWX/YYOHVo7fP3MmTPLkpKSnAMHDjyfnZ19aMaMGSeHDh2aBAB333132c0333yhqKjosgt57969q5cuXVo8efLkG10uFyIiIqq2bt36WUMx5eTkhE2bNu1U3fKFCxd+mZWV5TtlypTYd95559CyZcuKp0+f3r+yslIA4Be/+EXpoEGDKl555ZXizMzMG2pqapCcnHze8zOdOnXKNz4+3unv76+rV68+CABZWVnRxcXFAaoq6enpZ4cNG3bhWvffH//4x24rVqzo6evrq4GBga6VK1ce9PHxgY+PD5YsWVIybty4+JqaGsyYMeNEamrqxavXeImoNunow9e+YZFiAKnXkoikpqZqXh4bTYiIvCEi+aqaanccBQUFxcnJybZ96SwqKvKfOHFiXEMD391zzz0xQ4YMOZ+VldWiMTqdzqQdO3bs9bxt0xT69OlzU15e3p6oqCjvOoI0o4KCgu7JycmO+pbZ2SKiANabe1fLVPVVz4UiMhvAbACIiYmxITwiImrvBgwYkNS5c2fXsmXLWrzvyu7du9vdyL/Xw85EJF1VS0WkJ4ANIrJXVTe7F5rE5FXAahGxK0giImrbEhISKhtqDdm1a1e7SwZKS0s/tTsGb9jWWVVVS83f4wDeBpBmVyxERNSsXC6Xi4+57qDMsW/w10W2JCIiEiwiIe55ABkACu2IhYiIml1hWVlZGJORjsflcklZWVkYrnCNt+vWTCSAt83vuP0ArFLVv9kUCxERNaPq6ur7jx079vtjx44NBB+k2dG4ABRWV1ff39AKtiQiqnoQQLId2yYiopaVkpJyHMAku+Og1omZKREREdnG7geaERG1T6pAdXXjp4gIYOhQuz8NUbNhIkJEzcPlAmpqmuZi3BYn1zUNQXJ1d9wBZGc3TV1ErRATEaLmwgux3UcA8PNr3BQU1Pg6GjtFRNi9F4maFRMRah1Ugaoq4Pz5y6fy8kvzVVX2X1y9mWwaPuEy7eFCfL2Tjw8g/LUoUWvHRISurqYGuHDhm0lCU081NVePxVuNvZgFBNh/QeWFmIjaMSYibZkqUFnZdImAZ+uD51RR4X1sPj5AcLD1jbru1L17/eVXmjp39j4p8OGPwoiIWjsmIs2loVaEhi721ztdz334wMD6L/bBwUCPHt4nCfXV06kTv40TEdFVdbxERNX6ht/ctxmauhXhSglCQ+9pqGWBLQVERNRKtO9EpKgImDz5m0nC9XQi9GxF8Lzwd+kC9OzZ+FaEoCC2IhARUYfTvhOR4GBgwIDGJwhsRSAiImoW7TsRiY4G3nzT7iiIiIioAfyaT0RERLZhIkJERES2YSJCREREtmEiQkRERLaxLRERkXEiUiQi+0XkcbviICIiIvvYkoiIiC+AlwGMB+AEcJeIOO2IhYiIiOxjV4tIGoD9qnpQVSsBrAbwPZtiISIiIpvYlYj0AfCFx+vDpqyWiMwWkTwRySsrK2vR4IiIiKhltNoHmqnqqwBeBQARKRORzxtRXXcAJ5oksKbFuLzDuLzDuLzTHuPq15SBEDUHuxKRUgB9PV5Hm7J6qWqPxmxMRPJUNbUxdTQHxuUdxuUdxuUdxkVkD7tuzfwLQJyIxIqIP4DpALJtioWIiIhsYkuLiKpWi8gjAN4H4AtgharusiMWIiIiso9tfURUNQdATgtt7tUW2o63GJd3GJd3GJd3GBeRDURV7Y6BiIiIOig+4p2IiIhsw0SEiIiIbNMmExER6SsiuSKyW0R2icijpjxcRDaIyGfmbzdTLiKy1Ixr84mIDPWoa5ZZ/zMRmdXIuAJF5GMRKTBx/dKUx4rINrP9P5tfCkFEAszr/Wa5w6OuJ0x5kYh8tzFxedTpKyI7RGRda4lLRIpF5FMR2SkieabM1uNo6usqImtEZK+I7BGR4XbHJSIJZj+5p7Mi8hO74zL1ZZlzvlBE/mT+LbSG8+tRE9MuEfmJKWvx/SUiK0TkuIgUepQ1WRwikmL+He0375Xr22NENlDVNjcBiAIw1MyHANgHa8ya3wJ43JQ/DuBZM387gL8CEADDAGwz5eEADpq/3cx8t0bEJQC6mPlOALaZ7f0fgOmm/BUAD5n5OQBeMfPTAfzZzDsBFAAIABAL4AAA3ybYb/MArAKwzry2PS4AxQC61ymz9TiaOv8HwP1m3h9A19YQl0d8vgCOwXpgld3nfR8AhwB09jiv7rX7/AIwEEAhgCBYHfM3ArjRjv0F4FYAQwEUNsd5DuBjs66Y945vivOME6eWmGwPoEk+BPAugNsAFAGIMmVRAIrM/DIAd3msX2SW3wVgmUf5Zes1MqYgANsBfBvWUxH9TPlwAO+b+fcBDDfzfmY9AfAEgCc86qpdrxHxRAP4O4DRANaZ7bSGuIrxzUTE1uMIIAzWhVVaU1x1YskA8I/WEBcuDdkQbs6XdQC+a/f5BSATwHKP1z8H8B927S8ADlyeiDRJHGbZXo/yy9bjxKm1T23y1own06w7BFbrQ6SqHjWLjgGINPMNjW1z1TFvriMeXxHZCeA4gA2wvtWdVtXqerZRu32z/AyAiOaIC8ALsP4TdpnXEa0kLgWwXkTyRWS2KbP7OMYCKAPwB7FuZf1eRIJbQVyepgP4k5m3NS5VLQXwHIASAEdhnS/5sP/8KgRwi4hEiEgQrJaGvmg9x7Gp4uhj5ps6PqIW0aYTERHpAuAtAD9R1bOey1RVYV3kWpSq1qjqYFgtEGkAEls6hrpEZCKA46qab3cs9UhX1aEAxgN4WERu9Vxo03H0g9WM/t+qOgRAOaymc7vjAgCYvhaTALxZd5kdcZm+Dd+DlcD1BhAMYFxLxlAfVd0D4FkA6wH8DcBOADV11rHtOLbGOIjs0GYTERHpBCsJeUNV15riL0UkyiyPgtUqATQ8to1XY954Q1VPA8iF1STdVUTcD4/z3Ebt9s3yMAAnmyGumwFMEpFiAKth3Z75XSuIy/1tGqp6HMDbsJI3u4/jYQCHVXWbeb0GVmJid1xu4wFsV9UvzWu74xoL4JCqlqlqFYC1sM651nB+LVfVFFW9FcApWP3J7N5fbk0VR6mZb+r4iFpEm0xETI/w5QD2qOrzHouyAbh7ks+C1XfEXX6P6Y0+DMAZ0yT6PoAMEelmvtVlmLLrjauHiHQ1851h9VvZAysh+X4Dcbnj/T6AD8w3o2wA082vC2IBxMHqjHZdVPUJVY1WVQesJv0PVHWm3XGJSLCIhLjnYe3/Qth8HFX1GIAvRCTBFI0BsNvuuDzchUu3ZdzbtzOuEgDDRCTI/Nt07y9bzy8AEJGe5m8MgCmwOmvbvb/cmiQOs+ysiAwz+/8ej7qIWj+7O6lczwQgHVYz5iewmlt3wrr/GwGrQ+ZnsHrIh5v1BcDLsPprfAog1aOuHwLYb6b7GhnXIAA7TFyFABaY8v6w/kPdD6s5PcCUB5rX+83y/h51PWXiLUIT9oAHMBKXfjVja1xm+wVm2gXgKVNu63E09Q0GkGeO5TuwfqXQGuIKhtV6EOZR1hri+iWAvea8/19Yv3yx/bwHsAVWUlQAYIxd+wtW4ngUQBWsFrd/b8o4AKSafX8AwEuo09GaE6fWPPER70RERGSbNnlrhoiIiNoHJiJERERkGyYiREREZBsmIkRERGQbJiJERERkGyYi1K6JSI1YI9UWiMh2ERlxlfW7isica6h3k4ikXmdMOe7nzRARdXRMRKi9u6Cqg1U1Gdagav91lfW7whodttmo6u1qPXmXiKjDYyJCHUkorMd8Q0S6iMjfTSvJpyLyPbPOMwBuMK0oi826PzPrFIjIMx71ZYrIxyKyT0RuqbsxEYkSkc2mrkL3OiJSLCLdReRBs2yniBwSkVyzPENEPjKxvSnWmEpERO0SH2hG7ZqI1MB6OmUgrOHSR6tqvhnjJEhVz4pIdwD/hPVI8X6wnjw70Lx/PKzh48eq6nkRCVfVr0RkE4B8Vf2piNwOYJ6qjq2z7Z8CCFTV34iIr9neObHG/ElV1RNmvU4APgDwWwAfwRqrZbyqlovIz2A9kXRRc+4nIiK7+F19FaI27YJaoyFDRIYDWCkiA2E9RvtpsUb7dcEaNj2ynvePBfAHVT0PAKr6lccy92CL+QAc9bz3XwBWmETjHVXd2UCMv4M13sp7Yo2U7ATwD2vYEPjDSk6IiNolJiLUYajqR6b1owessYl6AEhR1SrTShHoZZUV5m8N6vm3pKqbTaIzAcDrIvK8qq70XEdE7oXVCvOIuwjABlW9y8tYiIjaJPYRoQ5DRBIB+MIMGgfguElCRsFKBgDgHIAQj7dtAHCfiASZOsK92F4/AF+q6msAfg9gaJ3lKQDmA/g3VXWZ4n8CuFlEbjTrBItIvHeflIio7WCLCLV3nUXEfUtEAMxS1RoReQPAeyLyKaxRdvcCgKqeFJF/iEghgL+q6mMiMhhAnohUAsgB8OQ1bnskgMdEpArA17CGZ/f0CIBwALnmNkyeqt5vWkn+JCIBZr3/BLDP609ORNQGsLMqERER2Ya3ZoiIiMg2TESIiIjINkxEiIiIyDZMRIiIiMg2TESIiIjINkxEiIiIyDZMRIiIiMg2/w9FRNYAdctgzgAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiIAAAEWCAYAAABbt/wMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdeZzN9f7A8dd7DAYzY19mbMMwZsbOpJSKMCmSiJ+oqC6i1KXNLVeSitttIxVFJaRyFZUuCZfSZsluZBnbWMc+mPXz++PznXGMWc5ojmNm3s/H4zxmvt/zXd7ne77ne97n8/0sYoxBKaWUUsobfLwdgFJKKaWKLk1ElFJKKeU1mogopZRSyms0EVFKKaWU12giopRSSimv0UREKaWUUl5TYBMRETEiUi+H598TkX+6TA8WkUMickZEKl7G/kKcffpebsz5xXkNdb0dx5UiIh+JyNg8LJ/juXG1EZHvRKSft+Nwh4jUcs6/YpexbikR+VpETorIF56IT10eEekrIou8HYcqmnJNREQkVkSSRKRSpvlrnQt+iDP9kTN9Z6bl3nDm93em+4tIqnMxOyMiu0TkQxEJy7dXBRhjHjbGvOjsszjwOhBtjPE3xsR78svKeY0/5tO2lonI31znOa9hZ35sPz9cDUmaiLQWkZXe2r+7RGS0iMxwnWeMuc0Y87G3YsqJ8/nvkD5tjNnjnH+pl7G5u4GqQEVjTM98iK2EiMxxYjQi0jbT8yIi40Uk3nmMFxFxeb6ZiKwWkbPO32Z/NSZvyOv1JqvPqzFmpjEm2jMRKpUzd0tEdgH3pE+ISGOgdBbLbQPud1nOF+gF7Mi03M/GGH+gLNABOAesFpFG7oeevSx+rVUF/IBN+bF9lb/yKYHpDCzIh+0UGV5IHGsD24wxKXldMYdYfwTuBQ5m8dxAoBvQFGgC3AEMcrZXApgHzADKAx8D85z5HuMkRwW2JFopjzDG5PgAYoGRwO8u8/4NPAcYIMSZ95Ez/xBQ3pnXBfgOe7Ho78zrD/yYxX6+AebkEMdTwAEgDnjQ2Xc9l32/i/0iSsAmNx8BY4EwZ54BzgBLgOXOdIIz7/+y2F8x5/UcBXYCjzjr+DrPlwWmOjHtd/ZVDIgAzgOpzrZPOMuXdLa3xzlG7wGlXPZ3J/AHcAqbuHUCXnK2c97Z1tvOsq6vvSwwHTgC7HbeKx/XY+3s9zg2obwtl/f6H8BmZ/kPAT/nuY3AHS7LFneOTXPnNaUf3zNAa2ySO9KJ6bATY1ln3RBn+YecdZc789sAK4ETwF4unDMfAZOAb4HTwK9AaKbY1wAtsjg+bZxttXWmw4HvgWNADNDLZRu5HcufgLeBk8BWoL3Luv2x58lp5zj3zeL4dgKSgGTnOK1z5i8D/pZpP284x2EncL0zf69zLPu5bDPH8yrT/l23HY89Z0Oxn4l45/2cCZRzlv8ESMP+UDgDPO3y3qV/DoKB+c7x3A4MyGbfL2R67Q9xGedIDufuvvT32GXeSmCgy/RDwC/O/9HYz624PL8H6JTN9pcBrwC/YT+j84AKLs9fx4Vzd51rLM66LznH/hxQz5k31lnnDPA1UNE5/qeA37lwbb3omLueM2R/vekMrHW2tRcYnel1Zv689sfluow9537Hnuu/A9dn2veLzus5DSwCKjnP+WGTu3jnWPwOVM3te0YfRfuR+wL2y6kD9qIdgf2y3Yf9dZM5ERkLTAEGO/M+x5akuJOIPAgcyiaGTtiLbCOgDDCLSxORk8AN2IubX3o8zvNZfZAz1s9mnw9jv2xqAhWApVx8Af4SmOzEUwV7gRqU3WvEXvznO9sKwF54XnGea+XE39GJvzoQ7jy3DOdLKqvYsRfvec42Q7ClUg+5xJEMDHDet8HYRE6yec2x2IQj/TX/5HIMnwY+c1n2TmBDDsf3QewXU13AH5gLfJJp+enO8SuFPZ9OO+dLcexFuZnL+xvvHCdf7MV6tsu+gnD5Ukk/PtjzZi/Qyplfxpl+wNlOc+yXb6SbxzIFGObE93/Oe1bB2e4poIFLPA2zOcajgRmZ5mW8xy77ecB5z8ZivzgmYZOOaOc4+ed2XmWx7/RtD3VefynnOHV0tl0Zm6S/mfnz7zJ90XvtLP8O9jPXDJvE3eLOayeP50gu16msEpGTwLUu01HAaef/YcB3mZb/Bngim+0vw55j6deg/6S/FuznNR64Hfv57ehMV3ZZdw/Q0DnuxZ1527GJYFls8r8Ne631dV73hzl8vjKfM5mvN22Bxk48TbDXz245bC9jG9hz6ThwnxPLPc50RZd978D+yCvlTI9znhuEPQdLY8/flkBgbt8z+ijaj7wUEX6Cve3SEdiC/VBmZTpwv4iUA24GvnJz+3HYD0BWemE/lBuNMQnYC1pm84wxPxlj0owx593cZ056YS/Ie40xx7C/hgAQkarYi87fjTEJxpjD2C+E3lltyLkvPRAYZow5Zow5DbzssvxDwDRjzPdO/PuNMVtzC9C5BdUb+Icx5rQxJhZ4DXsBSbfbGPO+sff0P8Z+SVbNYbNvu7zml7hwS24GcLuIBDrT92HPiez0BV43xuw0xpzBlrT0zlTEPto5fueAPsBiY8ynxphkY0y8MeYPl2W/NMb8Zmyx/kzsl16624H/GmOMy7ye2ETxNmPMb868LkCsMeZDY0yKMWYt9gulp5vH8jD2nEg2xnyGTc47O8+lAY1EpJQx5oAx5q/cBtzlxJgKfIZNDMcYYxKNMYuwJQv13DivshJnjJnovP5zxpjtznmXaIw5gq1LdbM7QYpITWzy/4wx5rzzfn2Ay+3ZXOT1HMkrf2wyku4k4O8ct8zPpT8fkMP2PnG5Bv0T6OWcN/cCC4wxC5zP7/fAKux5me4jY8wm57gnO/M+NMbsMMacxJYc7zDGLHbO8S+wifJlMcYsM8ZscOJZD3yKm+8r9pz+0xjziRPvp9gfZXe4LPOhMWab8758zoXPYzL2R0Q9Y0yqMWa1MebU5b4OVTTk5R7xJ9hfP3WwyUaWjDE/ikhl7K2bb4wx51zqh+WkOrZ4NyvBwGqX6d1ZLLPXnZ3kQXCmbbruszb2V80Bl9fmk0MMlbG/EFa71pXD/mIA+0VzOfUbKjlxuMa2G3ss02XcOzfGnHX275/DNjO/5mBn3TgR+QnoISJfArcBj+ewneAs4vLl4iTIdV81ubQukSvXOgBnufg13I4tJXP1d2C6MWajy7zawLUicsJlni/23HbnWO7PlOzsBoKNMQki8n/Ak8BU5zg94U4ymY1DLv+fAzDGZJ7nT+7nVVYuOkedpPot4Ebsl7AP9tevO4KB9AQo3W5syYO76+flHMmrM0Cgy3QgcMYYY0Qk83Ppz58me5k/G8Wx501tbDLr+kVdHFuKmtW66TK/p1m9x5dFRK4FxmFLcEpgS7zcbamU+X2BHK4rXPx5/AT7WZ7t/BidATznknwpdQm3S0SMMbux975vxxah5mQG8AQ5JCxZuAtYkc1zB7And7paWYWYh325I6d97gUSsfdFyzmPQGNMw2xiOYq9sDR0Wb6ssRV207cXmk0cOb2uo9hfILUzxZldaZU7Mr/mOJfpj7G//npiKxyn7yerGOOyiCuFiy+2ruvldAyy5bSIuhlb78NVT6CbiLgmS3uB/7m8B+WMbQEyGPeOZXW5OKvOOD7GmIXGmI7YEqetwPvZhJyf52lu55U7+3/ZmdfYGBOIfX8lh+VdxQEVRMS1FCEv519ez5G82oStqJquKRcqrG8CmmR6P5uQc4X2zJ+NZOx7sBdbWuJ6XpUxxoxzWf6vvI4E569rA4FquWx7FvaWXU1jTFls3SHJYXlXmd8XcPN9dUoLXzDGRGLrmXTB/RIyVUTltfb2Q9j7vwm5LDcBewtneU4LiUgxEakjIhOx9zRfyGbRz4H+IhIpIqWB5/MWdpYOYe9NZ+dz4DERqSEi5YER6U8YYw5gK2i9JiKBIuIjIqEikl70eQiokV4D3xiThv1iekNEqgCISHURudVZfirwgIi0d7ZVXUTCc4vTKbr/HHhJRAJEpDYwHJsIXq5HnNdcAVuq9ZnLc18BLbAlIa5J5hHsrQnXOD8Fhjnvrz/2C+8zk32LiZlABxHpJSK+IlLRzeaUbYD1WRT/xgHtgcdFZLAz7xsgTETuE5HizuMaEYlw81hWwZ4TxUWkJ7bO1AIRqSoid4pIGWyCesY5Hlk5BITkR8sJN84rdwRg4z0pItWxlcIzx5vd+bcXW9nyFRHxE5Em2GuEu+dfXs+RS4hISRHxcyZLOHGkf+FOB4Y7xyQY++PoI+e5ZdgKno8523jUmb8kh93d63INGoOtXJ+Kfb13iMitzjXNT0TaikgNd19HTpxbZvud/RcTkQe5OGm/6HrjCMCWVp0XkVbYW5/psvq8ulqA/Zz0cT6L/wdEYj8/ORKRdiLS2LlldQqbrGX3WVAKyGMi4tzPXOXGcseMMT9kKsZ21dopGj2FvSAEAtcYYzZks73vgDexF4nt5HyxcNdo4GMROSEivbJ4/n1gIbYG/BouLQW6H1vkmd7CZA721zBOfJuAgyJy1Jn3jBP7LyJyClgMNHBe32/YyolvYO9T/48Lv0jeAu4WkeMiMiGLOIdifzHtxFYKngVMc+8QZGkWNsnaib1VktGRmHM/+D/Y23NzXeafxWkV4BzP65wY0m/n7cLW7B+a3U6NMXuwpW1PYG/R/cHFv2azk22zXWeb7YERIvI35xZCNLYORRy2eHk8ttgacj+WvwL1sb+CXwLuNsbEYz9Hw51tHsOW0Awma+nF4/EissaN15ebbM8rN72ATS5PYlslZT7PXwFGOu/rk1msfw+28mMctgL388aYxW7uO0/nSDZisKVC1bGf13Nc+OxMxlac3ICthP2tMw9jTBK2ae/92NYdD2IrcyblsK9PsInMQWzl3Mecbe3FVt5+Fvslvxeb0OVnM90BzjbjsZVeXfvMyep6MwQYIyKngVHYJBsn3qw+r7g8H48tyXjC2d/TQBdjzFFyVw17LTyFrUv4P3KuS6ZURisDpRCRWGxN/Gy/SERkFBBmjLn3igWWAxHZjE0INnt4P/2xx6aNJ/ejrk4isgzbSuYDb8eiVGHj9e7KVcHh3K55iItbkniNUxQ93dNJiFJKKc/RHv6UW0RkALbI+TtjTI51f64UY0xSpgqBSimlChi9NaOUUkopr9ESEaWUUkp5TYGoI1KpUiUTEhLi7TCUUqpAWb169VFjTGVvx6FUTgpEIhISEsKqVbm2GlZKKeVCRLLqhVqpq4remlFKKaWU12giopRSSimv0UREKaWUUl6jiYhSSimlvEYTEaWUUkp5jSYiSimllPIaTUSUUkop5TWaiCil1FUoORk+/BA++8zbkSjlWZqIKKXUVSQ5GaZNg/BwePBBmDXL2xEp5VmaiCil1FUgORk++AAaNICHHoLy5WH+fPjqK29HppRnaSKilFJelJQE778P9evDgAFQsSJ88w38/jvccQeIeDtCpTxLExGllPKCpCSYMsUmIAMHQtWq8O238Ntv0LmzJiCq6NBERCmlrqDERHjvPZuADBoEQUHw3Xfwyy9w++2agKiiRxMRpZS6AhIT4d13bQIyeDAEB8N//ws//wydOmkCooouX28HoJRShdn58zB1KowbB/v2wfXX20qpHTtq8qEUaCKilFIecf68TTjGjYP9++GGG2y/IO3bawKilCtNRJRSKh+dO2dbwYwfD3FxcOON8PHHcMstmoAolRVNRJRSKh+cO2dbwYwfDwcOwE03wYwZ0LatJiBK5UQTEaWU+gvOnoXJk+Ff/4KDB23iMWuW/auUyp0mIkopdRnOnrXNcP/1Lzh0CNq1g9mz4eabvR2ZUgWLJiJKKZUHCQm2Ge6rr8Lhw7by6Rdf2LogSqm800REKaXckJAA77xjE5AjR6BDB3j+eWjTxtuRKVWwaSKilFI5OHMGJk2Cf/8bjh6F6GibgFx/vbcjU6pw0EREKaWycPr0hQQkPh5uvdUmIK1bezsypQoXTUSUUsrFqVPw9tvw2mtw7BjcdhuMGgXXXeftyJQqnDQRUUopbAIycSK8/rpNQG6/3ZaAtGrl7ciUKtw0EVFKFWknT15IQI4fhy5dbAnINdd4OzKligZNRJRSRdLJk/DWW/DGG3DiBNxxh01AoqK8HZlSRYsmIkqpIuXECZuAvPmm/f/OO20C0qKFtyNTqmjSREQpVSQcP26Tj7fesqUh3brZBKR5c29HplTR5uPpHYhIMRFZKyLfONN1RORXEdkuIp+JSAlPx6CUKrqOHbMJR0gIjBljR8Fduxa+/FKTEKWuBh5PRIDHgS0u0+OBN4wx9YDjwENXIAalVBFz7BiMHGkTkBdfhI4d4Y8/YO5caNbM29EppdJ5NBERkRpAZ+ADZ1qAW4A5ziIfA908GYNSqmiJj4fnnrMJyEsv2Y7I1q2DOXOgaVNvR6eUyszTdUTeBJ4GApzpisAJY0yKM70PqJ7ViiIyEBgIUKtWLQ+HqZQq6I4etZ2Qvf22HRemZ0/45z+hUSNvR6aUyonHSkREpAtw2Biz+nLWN8ZMMcZEGWOiKleunM/RKaUKiyNHYMQIWwIyfjx07gwbNsBnn2kSolRB4MkSkRuAriJyO+AHBAJvAeVExNcpFakB7PdgDEqpQurwYTsOzDvvwNmz0Lu3rRMSGentyJRSeeGxEhFjzD+MMTWMMSFAb2CJMaYvsBS421msHzDPUzEopQqfw4fhqaegTh17K+bOO2HTJpg1S5MQpQqiK9FqJrNngOEish1bZ2SqF2JQShUwBw/CE0/YWzCvvw7du8PmzTBzJkREeDs6pdTluiIdmhljlgHLnP93AjqMlFLKLQcPwr/+Be+9B4mJ0LevvQUTFubtyJRS+UF7VlVKXZUOHLiQgCQnw7332ma59et7OzKlVH7SREQpdVWJi7OtX6ZMsQnIfffZBKRePW9HppTyBE1ElFJXhf37LyQgKSnQrx88+yyEhno7MqWUJ2kiopTyqn37YNw4+OADSE29kIDUrevtyJRSV4ImIkopr9i790ICkpYGDzwA//iHbZarlCo6NBFRSl1Re/bYBGTqVDDmQgISEuLtyJRS3qCJiFLqiti9G155BaZNs9MPPWS7Zq9d27txKaW8SxMRpZRHxcbaBOTDD+303/5mExAdy1IpBZqIKKU8ZNcuePll+Ogj8PGBAQNsAlKzprcjU0pdTTQRUUrlq507bQLy8cc2ARk0yCYgNWp4OzKl1NVIExGlVL7YsQNeegmmTwdfXxg8GJ55BqpX93ZkSqmrmSYiSqm/ZPt2m4B88gkULw6PPGITkOBgb0emlCoINBFRSl2WP/+EsWPt6LfFi8PQofD00xAU5O3IlFIFiSYiSqk82bbtQgJSsiQ89phNQKpV83ZkSqmCSBMRpZRbYmLgxRfh009tAjJsGDz1FFSt6u3IlFIFmSYiSqkcbd1qE5DZs8HPD4YPhyef1AREKZU/NBFRSmVpy5YLCUjp0jb5eOIJqFLF25EppQoTTUSUUhfZtMkmIJ9/bhOQp5+2CUjlyt6OTClVGOU5ERERH8DfGHPKA/Eopbxk40abgHzxBZQpYzshGz4cKlXydmRKqcLMx52FRGSWiASKSBlgI7BZRJ7ybGhKqSthwwbo2RMaN4bvvrMj4cbG2t5RNQlRSnmaW4kIEOmUgHQDvgPqAPd5LCqllMetXw933w1NmsDChTBypE1AXnoJKlb0dnRKqaLC3VszxUWkODYRedsYkywixoNxKaU85I8/YMwY+PJLCAyEf/4T/v53qFDB25EppYoidxORyUAssA5YLiK1Aa0jolQBsnatTUC++grKloXnn4fHH4fy5b0dmVKqKHMrETHGTAAmuMzaLSLtPBOSUio/rVkDL7wA8+dDuXIwerRNQMqV83ZkSinlfmXViiIyQUTWiMhqEXkLKOvh2JRSf8Hq1dC1K7RsCcuX29KQ2FhbEqJJiFLqauFuZdXZwBGgB3C38/9nngpKKXX5fv8dunSBqCj48UfbJDc21tYFKas/H5RSVxl364gEGWNedJkeKyL/54mAlFKX57ff7C2YBQtsxdOxY+2IuIGB3o5MKaWy526JyCIR6S0iPs6jF7DQk4Eppdzzyy9w221w7bXw66+2/4/YWHjuOU1ClFJXP3dLRAYAfwdmONM+QIKIDAKMMUYvd0pdYT//bEtAFi60/X688go88ggEBHg7MqWUcp+7rWb00qbUVWLlSpuALFpkez4dPx6GDAF/f29HppRSeef2WDMi0hW4yZlcZoz5xjMhKaWy8uOPNgFZvNgOQPevf8HgwZqAKKUKNrcSEREZB1wDzHRmPS4iNxhj/uGxyJRSAKxYYROQH36AKlXg3/+Ghx+2A9MppVRB526JyO1AM2NMGoCIfAysBTQRUcpD/vc/m4AsXQpVq8Jrr9kEpHRpb0emlFL5x91WMwCuXSBpbwRKeciyZdCuHbRtC1u2wBtvwM6dMHy4JiFKqcLH3RKRV4C1IrIUEGxdkREei0qpIsYYm4CMHm17QQ0KgjffhIEDoVQpb0enlFKe426rmU9FZBm2ngjAM8aYgx6LSqkiwhhYssTeglmxAoKDYcIE+NvfNAFRShUN7o41I0B7bD2R+UAJEWnlxnp+IvKbiKwTkU0i8oIzv46I/Coi20XkMxEp8ZdehVIFjDG29cuNN0KHDvbWy8SJsGOH7Q1VkxClVFHhbh2Rd4DWwD3O9GlgkhvrJQK3GGOaAs2ATiJyHTAeeMMYUw84DjyUp6iVKqCMsf1/tGkDHTvC7t0waRJs3w6PPgp+ft6OUCmlrix3E5FrjTGPAOcBjDHHgVxLMYx1xpks7jwMcAswx5n/MdAtL0ErVdAYY3tAvf56uPVW2LsX3nnHJiBDhmgCopQqutxNRJJFpBg2iUBEKgNp7qwoIsVE5A/gMPA9sAM4YYxJcRbZB1TPU9RKFRDGwHffQevW0KkTxMXBe+/Bn3/azshKlvR2hEop5V3uJiITgC+BKiLyEvAjtiVNrowxqcaYZkANoBUQ7s56IjJQRFaJyKojR464GaZSVwdj7Ci4110Ht98OBw/C5Mk2ARk0SBMQpZRK526rmZkishpbYVWAbsaYLXnZkTHmhNP8tzVQTkR8nVKRGsD+LJafAkwBiIqKMnnZl1LeYgx8+61tBbNqFYSEwJQp0K8flNAq2UopdQl3W808ZIzZaoyZZIx52xizxen2Pbf1KotIOef/UkBHYAuwFLjbWawfMO/ywlfq6mAMfP01XHMN3HEHHD0KH3wA27bBgAGahCilVHbcvTXTQ0T6pk+IyCSgshvrBQFLRWQ98DvwvTNY3jPAcBHZDlQEpuYtbKWuDsbA/PkQFQVdu8Lx4zB1qk1AHnoIihf3doRKKXV1c7dn1R7AfBFJAzphK5vm2uTWGLMeaJ7F/J3Y+iJKFUjGwLx5MGYMrF0LoaHw4YfQt68mH0oplRc5loiISAURqQCUAv4GPI3tQ+QFZ75SRUpaGnz5JTRvDnfdBadPw0cfwdat0L+/JiFKKZVXuZWIrMY22RWXv52dhwHqejQ6pa4S6QnImDGwfj3Urw/Tp8M994Cvu+WKSimlLpHjJdQYU+dKBaLU1SgtDebOtQnIhg0QFgaffAK9e2sCopRS+cHdyqpKFSlpafD559C0KfTsCUlJMHMmbN4M996rSYhSSuUXTUSUcpGaCp99Bo0bw//9n52eNQs2bYI+faBYMW9HqJRShUtulVW16p0qElJT4dNPbQLSu7edN3u2vR1zzz2agCillKfkViLys4h8JSIPi0jIFYhHqSsqvcSjUSNb4uHjY0tENmywJSKagCillGflmIgYY6KAvzuTb4rI7yLyhohEi4iOlqEKrJQUmDEDGja0fX/4+sIXX9gWMb162YREKaWU5+V6uTXGxBpj3jPGdAOuB74GOgArRORbTweoVH5KSbGtXiIj4b777OBzc+bAunVw992agCil1JWWp7r/xphkYInzQESqeyIopfJbSopt9TJ2LGzfblvDzJ0Ld96pyYdSSnnTX7oEG2MuGTVXqatJSort+TQ83PZ86u9vOyZbs8b2jKpJiFJKeZf2hqAKpeRkewvmpZdg507bJfu8eXZkXBFvR6eUUiqdJiKqUElOtl2vv/QS7NoFLVva0XG7dNEERCmlrkZuJSIiEgY8BdR2XccYc4uH4lIqT5KSLiQgsbEQFQUTJkDnzpqAKKXU1czdEpEvgPeA94FUz4WjVN4kJdk6IC+/DLt3wzXXwKRJcNttmoAopVRB4G4ikmKMedejkSiVB0lJ8OGHNgHZsweuvRbefRc6ddIERCmlCpLcunivICIVgK9FZIiIBKXPc+YrdUUlJtqEo149ePhhCA6G//4Xfv5ZS0GUUqogyq1EZDVggPTL+1MuzxmgrieCUiqzxESYOhVeeQX27YPrr4cPPoCOHTX5UEqpgizHRMQYU+dKBaJUdubPh0cfhb174YYb7C2Z9u01AVFKqcLAre6cROQRESnnMl1eRIZ4LiylIC7Odrt+551Qrhx8/z2sWAEdOmgSopRShYW7/UoOMMacSJ8wxhwHBngmJFXUpaXB5Ml2PJhvvrEVUlev1gREKaUKI3dbzRQTETHGGAARKQaU8FxYqqjavBkGDoSffoJbboH33oP69b0dlVJKKU9xt0RkIfCZiLQXkfbAp8B/PReWKmoSE+H556FZM9iyxdYDWbxYkxCllCrs3C0ReRoYCAx2pr8HPvBIRKrIWbECBgyAmBjo2xdefx2qVPF2VEoppa6EXBMR5zbMdGNMX2zvqkrlixMn4JlnYMoUCAmB776zHZIppZQqOnK9NWOMSQVqi4jWCVH5whj44guIiLB9gTzxBGzcqEmIUkoVRe7emtkJ/CQi84GE9JnGmNc9EpUqtPbuhSFDbGuYFi3g22/tX6WUUkWTu4nIDufhAwR4LhxVWKWm2sHonnvONs997TV47DHwdfcMVEopVSi59TVgjHkBQET8nekzngxKFS7r19vKqL/9BrfeaseKqaN99iqllML9nlUbichaYBOwSURWi0hDz4amCrpz5+Af/4CWLWHXLqqpA78AACAASURBVJg501ZI1SREKaVUOncLxqcAw40xSwFEpC3wPnC9h+JSBdwPP8CgQbBjBzzwALz6KlSs6O2olFJKXW3c7dCsTHoSAmCMWQaU8UhEqkCLj4f+/S90x/7DDzBtmiYhSimlsuZuIrJTRP4pIiHOYyS2JY1SgG2SO3MmhIfbv88+a+uG3HKLtyNTSil1NXM3EXkQqAzMBf4DVAIe8FRQqmDZtQtuuw3uvRdCQ+0AdS+9BKVKeTsypZRSVzt364h0MMY85jpDRHoCX+R/SKqgSEmBN9+EUaOgWDGYOBEGD7b/K6WUUu5wt0TkH27OU0XE6tXQqhU89RR07GhHzX30UU1ClFJK5U2OJSIichtwO1BdRCa4PBUIpHgyMHV1SkiwJSBvvmkHppszB7p3txVTlVJKqbzKrUQkDlgFnAdWuzzmA7fmtKKI1BSRpSKyWUQ2icjjzvwKIvK9iPzp/C3/11+GuhK++w4aNrSj4w4YAFu2QI8emoQopZS6fDmWiBhj1gHrROQ48I0xJi0P204BnjDGrBGRAGC1iHwP9Ad+MMaME5ERwAjgmcsLX10Jhw7BsGHw6ae2VcyKFdCmjbejUkopVRi4W0ekF/CniPxLRMLdWcEYc8AYs8b5/zSwBagO3Al87Cz2MdAtbyGrK8UY2wdIRAT85z8wejT88YcmIUoppfKPW4mIMeZeoDl24LuPRORnERnolHTkSkRCnPV/BaoaYw44Tx0EqmazzkARWSUiq44cOeLOblQ++vNP2wfIQw/Z2zF//AHPPw8lS3o7MqWUUoWJuyUiGGNOAXOA2UAQcBewRkSG5rSeM1Def4C/O9tw3aYBTDb7m2KMiTLGRFWuXNndMNVflJQEL78MjRvD2rUweTL873+2VEQppZTKb271IyIiXbEdmNUDpgOtjDGHRaQ0sBmYmM16xbFJyExjzFxn9iERCTLGHBCRIODwX30RKn/88outhLpxI9x9N0yYAEFB3o5KKaVUYeZuiUgP4A1jTGNjzKvGmMMAxpizwENZrSAiAkwFthhjXnd5aj7Qz/m/HzDvsiJX+ebUKdsHyPXXw4kTMG8efPGFJiFKKaU8z60SEWNMPxGp5pSMGOB3Y8xB57kfslntBuA+YIOI/OHMexYYB3wuIg8Bu7EVYZWXzJsHjzwCcXEwdCiMHQsBbtX8UUoppf46d2/NPAQ8DywBBJgoImOMMdOyW8cY86OzbFba5zVQlb/i4uCxx2xrmMaN7d9rr/V2VEoppYoad8eaeRpoboyJBxCRisBKINtERF2d0tJgyhR45hlITLQVU598EooX93ZkSimliiJ3E5F44LTL9GlnnipANm+GgQPhp59s09z33oP69b0dlVJKqaLM3URkO/CriMzD1hG5E1gvIsMBMlVGVVeZ9JKPV16x9T8++gjuv1+7ZlfqapaUmsTBMwdJM2mElAvxdjhKeYy7icgO55EuvaWLVmu8yi1fbktBYmKgb187TkyVKt6OSqmiKzk1mUMJh4g7HZfxOHD6gP3/zIXpI2dtR453NriTr3p/5eWolfIcd1vNvODpQFT+OnECnn4a3n8fQkLsgHWdOnk7KqUKr5S0FA4nHL4owchIMs5cmD6ScASTqR9HH/Ghmn81ggOCqV22Nq1rtCY4IJjggGAiKmlvgqpwc7dERBUQxsCcObZFzOHDtiLq6NFQpoy3I1OqYEpNS81IMA6cOXBxkuEyfejMoUsSDEGo6l+V4IBgagTWoFVwK4ICgjKSjOCAYIL8g6hSpgrFfIp56RUq5V2aiBQie/fCkCHwzTfQogV8+639q5S6VJpJ40jCkSyTCtd56fU0XAlClTJVMpKKFtVa2KQiU5JRpUwVfH30MqtUTvQTUgikpsKkSfDcc7Z57muv2RIRX313VRGUZtKIPxufZVLhOn0o4RApaSmXrF+5dOWMpKJp1aYXSi5ckoyqZapSvJi2eVcqP7jboVkY8C525NxGItIE6GqMGevR6FSu1q+348P89putA/LOO1CnjrejUir/GWOIPxd/ceXOLJKMA2cOZJlgVCxVMSORaFilIcH+wRffIgkIopp/NUoUK+GFV6dU0eXub+b3gaeAyQDGmPUiMgvQRMRLzp2DMWPg3/+G8uVh1izo3Vub5KqCxxjD8fPHL21BkqkVyYEzB0hKTbpk/fJ+5TOSifBK4QT5X3x7JDggmGr+1SjpW9ILr04plRt3E5HSxpjf5OJvuUt/cqgrYvFiePhh2LEDHnjAJiMVKng7KqUuZozhxPkTl1bwzNSK5MDpAySmJl6yfjm/chlJxU21b7qocqdrglGqeCkvvDqlVH5xNxE5KiKh2M7MEJG7gQMei0plKT4enngCPv4Y6tWDH36wPaQqdSUZYziVeCrXViRxp+M4n3L+kvUDSwZmJBI31LzhkhYk6bdJShcv7YVXp5S60txNRB4BpgDhIrIf2AXc67Go1EWMsbde/v532z/Is8/CyJFQSn8Iqnx2OvF0tkmF6/TZ5LOXrOtfwj8jobiuxnWX3CIJ8g8iKCAI/xL+XnhlSqmrlbsdmu0EOohIGcDHGHM6t3VU/ti5EwYPhkWL7Oi4779vR8tVKi/OJJ3JsYJn+vSZpDOXrFu6eOmMZCIqOIpg/0ubqQb5BxFQUjtaVkrlnbutZsoB9wMhgG96XRFjzGMei6yIS0mBN96A55+HYsVg4kSbkBTTPo+Ui7PJZ3NtRRJ3Oo7TSZf+dvDz9aN6QHWCAoJoHtSczv6ds+wLI6BEAKK1oJVSHuLurZkFwC/ABiAtl2XVX7R6Nfztb/DHH9C1K7z9NtSs6e2o1JV0LvlcRjKRuRWJ6/TJxJOXrFuyWMmMJKJx1cbcGnrrJc1UgwOCKVuyrCYYSimvczcR8TPGDPdoJIozZ2wJyJtv2oHp5syB7t21SW5hkpiSeHGfF5kGO0ufd/z88UvWLe5TPCOZiKwcSYe6HS6thxEQRHm/8ppgqKvK6tWrq/j6+n4ANAJ8vB2PuqLSgI0pKSl/a9my5eGsFnA3EflERAYA3wAZ7eyMMcf+eowK7KB0gwfD7t0waBCMGwflynk7KuWupNSkjL4ucrpFcuzcpR+Z4j7FMwY8a1CxAe1C2l3STDU4IJgKpSpogqEKJF9f3w+qVasWUbly5eM+Pj4m9zVUYZGWliZHjhyJPHjw4AdA16yWcTcRSQJeBZ6DjFGdDFD3L0dZxB06ZFvDzJ4NERGwYgW0aePtqFS65NRkDp45mGsrkqNnj16ybjEplnEbJLR8KDfWujHLvjAqlq6Ij+iPRFWoNdIkpGjy8fExlStXPnnw4MFG2S3jbiLyBFDPGHPp1VZdFmPgww/t6LgJCfDCC/DMM1BSO3+8IlLSUjh05lCuA57lNmR7SLmQi4Zsd00yKpeprAmGUpaPJiFFl/PeZ3sxdDcR2Q5c2nGAuizbttnbL8uW2dKPKVNsaYj661yHbM/pFsnhhMNZJhhVylQhOCCYmmVrcm31ay9pQRIcEEzl0pV1yHallMon7iYiCcAfIrKUi+uIaPPdPEhKgldfhRdfBD8/mDzZto7x0R/NuUpNS+XI2SO5NlU9lHAo1yHbWwa11CHblVI5GjFiRLVx48YdzI9tDR8+PNjf3z91zJgxh7J6vk+fPrX69+8fP3LkyBr//ve/9950001X3Q//d999t8Ibb7xRDaBMmTJp77zzzu7WrVufy49tu3vV/cp5qMv0yy92lNyNG6FnT3jrLQgK8nZUVy9jDDHxMSzasYhFOxaxLHYZCckJlyyX3ZDtromGDtmulMqrCRMmBOUlEUlLS8MYQ7HL6OxpzZo1/tOnT9+T5xWvoHr16iX+9NNPMZUrV079/PPPAwcNGlR7/fr1W/Nj2+72rPpxfuysKDp1ynbJ/s47UL06zJ8Pd9zh7aiuTsfOHeOHnT/Y5GPnIvactJ/LehXq0a9pPyIrR+qQ7UqpPIuJiSnRqVOn+o0bNz67cePG0mFhYee++OKL2KVLl5aZMGFClcWLF+8A+PLLLwPfeeedyvXr1z+fmJjoEx4eHhkWFnZu/vz5u0aPHl115syZlQDuu+++I6NGjTocExNT4tZbbw1r3rz5mQ0bNpRZsGDBn+vXr/cbNWpU9dTUVKlQoULKzz//vA1gy5YtpVq1atUgLi6uxMMPP3xo5MiRhwHWrFnjV7du3fO+vhe+jlNTU+nVq1dI9erVkyZMmBA3d+7cwDFjxgQnJSVJ7dq1E2fPnh1btmzZtHnz5gWMGDGiZmpqKk2bNj07ffr03aVKlTLVq1dvfMcddxxfsmRJYMmSJc2nn366s1GjRonTpk0r/8orrwT7+PiYgICA1FWrVsW4eww7duyY8UuwXbt2CY8++mi+XXxzTERE5HNjTC8R2QBcUtHIGNMkvwIpjObNg0cegbg4GDoUxo6FAO0FO0NyajK/7v+VhdsXsmjnIn7f/zsGQ9mSZWlftz3PtnmWjqEdqVteG2cpVVg8+CA1N24kX0c0bNSIs9OmsTenZWJjY/0mT54cGx0dndCzZ8+QV199tfLo0aMPPf7447Xi4uJ8g4ODU6ZNm1bxgQceONqnT5+TH330UZWtW7duBlixYkXpWbNmVVy9evUWYwwtW7aMaN++/elKlSql7tmzp+TUqVN3tW/fPjYuLs730UcfDVm2bNnW8PDwpEOHDmUUj2zfvt1v5cqVMSdOnCgWERHR6KmnnjpSsmRJM3/+/LLR0dEZPRMmJydLt27d6kRGRp4bP378wQMHDvi+/PLLQcuXL98WGBiY9txzz1V78cUXq44ZM+bgoEGD6ixatCimSZMmiXfddVfIq6++WnnUqFGHAcqWLZuybdu2zW+//XbFoUOH1ly6dOn2cePGBS1atGhbnTp1ko8ePXrZFd0mTpxYqV27dpf2pniZcisRedz52yW/dlgUpCcec+facWH+8x87ToyC7ce2Z9xuWbJrCaeTTuMjPlxb/VpG3TyK6NBoWlVvpXU1lFL5qlq1aknR0dEJAPfdd1/8hAkTqvj4+Bzq1atX/Pvvv1/hkUceiV+zZo3/3Llzd2Ved9myZf633377icDAwDSAzp07H1+6dGlAz549TwQFBSW1b98+wVmuTKtWrU6Hh4cnAVStWjU1fRvR0dEnSpUqZUqVKpVSoUKF5H379vmGhoYmL168OHDGjBmx6csNGTKkdrdu3Y6NHz/+YPo2d+zY4deqVatwsIlKy5Ytz6xbt86vRo0aiU2aNEkE6N+/f/ykSZOqAIcB+vXrdwxgwIABx0aOHFkTICoq6kzfvn1DevTocbxv376X9prohq+//jpgxowZlVauXJkvt2Ugl0TEGHPA+XeIMeYZ1+dEZDzwzKVrFV1pabYFzDPP2Iqpr7wCTzwBxYtw9YST50+yZNeSjNstO4/vBKB22drc0+gebq13K7fUuYVyftp7m1JFQW4lF56SuTPA9OnBgwfHd+7cuZ6fn5+54447jhfP4wW7dOnSbg17UrJkyYy7CsWKFSMlJUVOnz7tc+rUqWIhISHJ6c9FRUWdWbFiReDZs2cPlS5d2hhjaNOmzamvv/76ogTp559/znH8dR+XVhAiYgBmzZq1Z8mSJWXmz59ftmXLlpGrV6/eXK1atYxkaejQodW///77sgDppUGufv3111JDhgyp/e233/7put5f5W57jY5ZzLstv4IoDDZvhptusr2jRkXBhg0wYkTRS0JS0lL4Zd8vjPnfGG6YdgMV/1WR7p93Z8aGGTSq0oi3b3ubbY9uY9fju5h8x2S6R3TXJEQp5XEHDhwosXjx4jIAM2fOrHD99defAQgJCUmuWrVq8muvvRY0cODAjL6yfH19TWJiogC0a9fuzIIFC8o5iYPPggULyrdr1+6SkSTbtm2b8NtvvwVs3bq1BIDrrZmsfPvttwFt2rS5aDuDBg06Gh0dfbJLly6hycnJtG3bNmHVqlX+GzduLAlw6tQpn/Xr15ds2rTp+f3795dInz99+vSKN954Y8a2pk+fXgFg6tSp5Zs3b54AsGnTppK33HJLwptvvhlXvnz5lJ07d15Uz2PixIn7t27dujmrJOTPP/8s0bNnz9Bp06btSi+FyS+51REZDAwB6orIepenAoCf8jOQgioxEV5+2ZZ+BATARx/B/fcXrfFhdp/YzcIdC1m0YxE/7PqBE+dPIAhRwVGMaDOC6NBoWtdorS1XlFJeExIScn7ixIlVBg4cWLp+/frnn3zyySPpz/Xu3Tt+0qRJvi1atDifPq9v375HIiIiIhs1anR2/vz5u/r06RPfokWLCLCVVW+44YZzMTExF32RBwcHp0yYMCH2rrvuqpeWlkbFihWTV65c+Wd2MS1YsKBsr169LrlFMnr06EPDhg0r1r179zpfffXVrsmTJ8f27t27blJSkgA8//zz+5s0aZL43nvvxfbs2TM0vbKq62s6fvx4sbCwsMgSJUqY2bNn7wQYNmxYjdjY2JLGGGnTps2p6667zu3mtyNHjgw6ceKE79ChQ2uDTdQ2bty4xd31cyLGZN/ZnYiUBcoDrwAjXJ46fSXHmYmKijKrVq26Urtz2/LlMHAgxMRA377w+ut2sLrC7nTiaZbFLsu43bItfhsANQJrEF03mlvr3Ur7Ou2pWLqilyNVqmgTkdXGmChvx7Fu3brYpk2beq1n7piYmBJdunSp/+eff27K6vn777+/VvPmzc8OGzbsisYYGRkZsXbt2q2ut23yQ/Xq1RuvWrVqS1BQUEp+bvevWLduXaWmTZuGZPVcbnVETgIngXs8EFeBdfy4rQfy/vsQEgL//S/cequ3o/KcNJPGmgNrMiqZrty7kuS0ZEoXL03bkLYMiRpCdGg04ZXCdVA2pVSB0rBhw4hSpUqlTZ48+YrXXdm8eXO+lCgUdNo0IQ+MgS++gMcegyNH7Dgxo0dDmTLejiz/7Tu1j+93fM/CHQtZvHMx8efiAWherTnDWw8nOjSaG2reQElfHRxHKXV1a9CgQVJ2pSGbNm0qdMnA/v37N3g7hrzQRMRNe/bYPkG++QZatIAFC+zfwiIhKYHlu5dn3G7ZfMTWVarmX43OYZ2JrhtNx9COVClTBO49KaWUumI0EclFaipMmgTPPWeb5772mi0R8S3gRy7NpLH+0PqM2y0r9qwgKTUJP18/bqp9Ew82e5Do0GgaVWmkt1uUUkp5TAH/OvWsdevs+DC//w6dOsG779o6IQXVwTMH+X7H9yzauYjvd3zPoQQ7/lLjKo0Z2moo0aHR3FjrRkoVz7F5ulJKKZVvNBHJwrlzMGaMHSm3QgWYNQt69y54TXLPp5znxz0/ZnShvv6QbYFduXRlOoZ2zLjdEhwQ7OVIlVJKFVU6AH0mixfbbtnHjYN+/WDrVrjnnoKRhBhj2Hh4I6///DqdZnSi/PjydPykI2/9+hYVS1VkXPtxrBm4hoNPHmRm95n0a9ZPkxCllMrCiBEjquXXtoYPHx48atSoqtk936dPn1qLFi0q06pVqwbLly/P13F48svatWv9mjVrFl6iRIkWmV/LnDlzAkNCQhrVqlWr0bPPPpvn4+bREhERmYYdp+awMaaRM68C8BkQAsQCvYwxl9XnfX46etS2gvn4Y6hXD5YsgXbtvB1V7o4kHGHxzsUs2mnresSdjgMgolIEg1oOIjo0mptr30yZEoWwaY9SSnnIhAkTgsaNG3fQ3eXT0tIwxlCsWN7HkluzZo3/9OnT9+R5xSuoSpUqKW+99daeOXPmlHedn5KSwrBhw2otXLhwW926dZObNm0a0aNHjxMtW7Y8n922MvN0ichHQKdM80YAPxhj6gM/cHFHaVecMTBjBkREwMyZ8OyzsH791ZuEJKUmsSx2Gc/+8Cwtp7Skyr+r0GduH+bHzKdNrTZ8cMcH7P77bjY/spk3O73J7fVv1yREKVWkxcTElKhTp07Drl271qlbt27DTp061T19+rTP/PnzAzp06BCavtyXX34Z2LFjx9AhQ4ZUT0xM9AkPD4/s2rVrHYDRo0dXrV+/fsP69es3HDNmTJX07YaEhDS66667QsLCwhru2LGjxJw5cwIjIyMjGjRoENm6deuw9G1v2bKlVKtWrRrUqFGj8dixYzOaH65Zs8avbt26531dWkCkpqbSo0ePkMceeywYYO7cuYHNmjULj4yMjLjtttvqnjx50gdg3rx5AREREZFhYWGRPXv2DDl37pyA7dDs4YcfrhEWFhbZuHHjiPRu4KdNm1a+fv36DRs0aBAZFRXVIC/HsHr16ik333zz2eLFi1/U+dqyZcvK1K5dOzEyMjLJz8/PdO/e/dicOXPyNG6HR0tEjDHLRSQk0+w7gbbO/x8Dy/DS4Hk7d9qxYRYtsqPjvv++vS1zNTHGEBMfk9G6ZVnsMhKSE/D18aV1jdaMbTeW6NBoWgS1oJjPZY/qrJRSV8SD8x6sufHwxny9/dCoSqOz0+6clmOHZLGxsX6TJ0+OjY6OTujZs2fIq6++Wnn06NGHHn/88VpxcXG+wcHBKdOmTav4wAMPHO3Tp8/Jjz76qEr6mCsrVqwoPWvWrIqrV6/eYoyhZcuWEe3btz9dqVKl1D179pScOnXqrvbt28fGxcX5PvrooyHLli3bGh4enuQ61sz27dv9Vq5cGXPixIliERERjZ566qkjJUuWNPPnzy8bHR19Mn255ORk6datW53IyMhz48ePP3jgwAHfl19+OWj58uXbAgMD05577rlqL774YtUxY8YcHDRoUJ1FixbFNGnSJPGuu+4KefXVVyuPGjXqMEDZsmVTtm3btvntt9+uOHTo0JpLly7dPm7cuKBFixZtq1OnTvLRo0fz5Qtj7969JapXr56UPl2jRo2kX3/91T8v2/BGZdWqLqP6HgSyvG8mIgOBgQC1atXK1wBSUuCNN+D556FYMZg40SYkl1Gi5hHHzh3jh50/ZPTpseekLbGrV6Ee/Zv1Jzo0mrYhbQksGejlSJVSqmCoVq1aUnR0dALAfffdFz9hwoQqPj4+h3r16hX//vvvV3jkkUfi16xZ4z937txdmdddtmyZ/+23334iMDAwDaBz587Hly5dGtCzZ88TQUFBSe3bt09wlivTqlWr0+Hh4UkAVatWzRihNjo6+kSpUqVMqVKlUipUqJC8b98+39DQ0OTFixcHzpgxIzZ9uSFDhtTu1q3bsfHjxx9M3+aOHTv8WrVqFQ42UWnZsuWZdevW+dWoUSMxfQC6/v37x0+aNKkKcBigX79+xwAGDBhwbOTIkTXBjuzbt2/fkB49ehzv27ev16tEpPNqqxljjEkfnjiL56YAU8CONZNf+1y1yjbJ/eMP6NoV3n4batbMr61fnuTUZH7d/2tGqcfvcb+TZtIoW7Is7eu259k2z9IxtCN1y9f1bqBKKfUX5VZy4SmZ+0NKnx48eHB8586d6/n5+Zk77rjjePE8DpleunTpNHeWcx1PplixYqSkpIgzmm+xkJCQ5PTnoqKizqxYsSLw7Nmzh0qXLm2MMbRp0+bU119/fVGC9PPPP+fYz4KPz4WaF+nfs7NmzdqzZMmSMvPnzy/bsmXLyNWrV2+uVq1aRrI0dOjQ6t9//31ZgKxG4M1KzZo1k/bv358x+N++ffsuKiFxhzdazRwSkSAA5+/hK7HTM2dg+HB7C+bQIZgzB776yntJyI5jO3j393fpNrsbFf9VkRs/vJGXVryEj/jwz5v+ycoHV3L06aP8p9d/GBQ1SJMQpZT6Cw4cOFBi8eLFZQBmzpxZ4frrrz8DEBISkly1atXk1157LWjgwIEZg975+vqaxMREAWjXrt2ZBQsWlHMSB58FCxaUb9eu3enM+2jbtm3Cb7/9FrB169YSAK63ZrLy7bffBrRp0+ai7QwaNOhodHT0yS5duoQmJyfTtm3bhFWrVvmn1/M4deqUz/r160s2bdr0/P79+0ukz58+fXrFG2+8MWNb06dPrwAwderU8s2bN08A2LRpU8lbbrkl4c0334wrX758ys6dOy8aPXjixIn7t27dutndJATg5ptvToiNjfXbunVrifPnz8vcuXMr9OjR44S764N3SkTmA/2Acc7feZ7e4Xff2Vsvu3fDww/DK69AuTxVpfnrTp4/yZJdSzJut+w8vhOAkHIh9Gnch+jQaG6pcwvl/K5wYEopVQSEhIScnzhxYpWBAweWrl+//vknn3zySPpzvXv3jp80aZJvixYtMlp69O3b90hERERko0aNzs6fP39Xnz594lu0aBEBcN999x254YYbzsXExFz0RR4cHJwyYcKE2LvuuqteWloaFStWTF65cuWf2cW0YMGCsr169brkFsno0aMPDRs2rFj37t3rfPXVV7smT54c27t377pJSUkC8Pzzz+9v0qRJ4nvvvRfbs2fP0NTUVJo2bXrW9TUdP368WFhYWGSJEiXM7NmzdwIMGzasRmxsbEljjLRp0+bUddddd87d47dnzx7fa665JjIhIaGYiJjJkydX3bJly8YKFSqkvfbaa3s6deoUlpqaSp8+fY5GRUW53WIGQIzJ19GHL964yKfYiqmVgEPA88BXwOdALWA3tvnusZy2ExUVZVatWpXn/R8+DI8/DrNn21YxU6ZAmzZ53sxlSUlLYVXcqozbLb/s+4VUk4p/CX9uqXML0XWjiQ6Npl6FetqFulLKI0RktTEmyttxrFu3LrZp06ZHc1/SM2JiYkp06dKlfnYD391///21mjdvfnbYsGFXNMbIyMiItWvXbnW9bZMfqlev3njVqlVbgoKCUvJzu3/FunXrKjVt2jQkq+c83Wrmnmyeau/J/aY7ccIOTvfCC/DMM1DSwwPF7j6xm0U7FrFwx0J+2PUDJ86fQBCigqMY0WYE0aHRtK7RmuLF8nYPUimllGc0bNgwolSpUmmTJ0++4nVXNm/eXOhG/r0chbqL97AwO2pu2bKe2f6ZpDMs3vYSvQAAFQVJREFU3bU043bLtvhtANQIrEGPiB5Eh0bTvk57Kpau6JkAlFJK5apBgwZJ2ZWGbNq0qdAlA/v379/g7RjyolAnIpC/SUiaSWPNgTUZt1tW7l1JcloypYuXpm1IW4ZEDSE6NJrwSuF6u0UppZRyQ6FPRP6qfaf2XTRibfy5eACaV2vO8NbDiQ6N5oaaN1DS18P3fZRSSqlCSBORTM4mn2X57uUZdT02H7GtmIL8g+gS1oXo0Gg61O1AlTJVctmSUkoppXJT5BORNJPGhkMbWLhjIYt2LGLFnhUkpSbh5+vHTbVv4sFmDxIdGk2jKo30dotSSimVz7zRoZnXHTxzkE/WfcJ9X95H8GvBNJvcjGcWP8PhhMMMbTWUhfcu5NjTx1h470KeuP4JGldtrEmIUkoVISNGjMjzcPbZGT58ePCoUaOyHM4EoE+fPrUWLVpUplWrVg2WL1+er+Pw5JdvvvkmICAgoFl4eHhkeHh45JNPPhmU/tycOXMCQ0JCGtWqVavRs88+m+fjViRKRM6nnOfHPT9m3G5Zf2g9AJVLV6ZjaEduDb2VDnU7EBwQ7OVIlVJKXQ0mTJgQNG7cuIPuLp+WloYxhmKXMWjZmjVr/KdPn74nzyteYVFRUWeWLl263XVeSkoKw4YNq7Vw4cJtdevWTW7atGlEjx49TrRs2dLtTs0KdYlIzNEYOs3oRPnx5en4SUfe+vUtKpWuxLj241gzcA0HnzzIzO4zub/p/ZqEKKVUIRUTE1OiTp06Dbt27Vqnbt26DTt16lT39OnTPvPnzw/o0KFDaPpyX375ZWDHjh1DhwwZUj0xMdEnPDw8smvXrnUARo8eXbV+/foN69ev33DMmDFV0rcbEhLS6K677goJCwtruGPHjhJz5swJjIyMjGjQoEFk69atw9K3vWXLllKtWrVqUKNGjcZjx47NqGS4Zs0av7p165739b1QLpCamkqPHj1CHnvssWCAuXPnBjZr1iw8MjIy4rbbbqt78uRJH4B58+YFRERERIaFhUX27Nkz5Ny5cwK2Q7OHH364RlhYWGTjxo0j0ruBnzZtWvn69es3bNCgQWRUVFSD/Di2y5YtK1O7du3EyMjIJD8/P9O9e/djc+bMyVMX4YW6RKRCqQrsPbWXQS0HER0azc21b6ZMiTLeDksppYquBx+sycaN+Xv7oVGjs0zLeTC92Nj/b+/eo7Oo7zyOv79JCCFcAgEM4aKhikDKgkAOi4gtiJsqYCsqFmEV3bW43o7F6rauu9baU1fL4rFezkot1GUrpWvXddHSFahwpMVigxgFTBAhRW5ykYtyzeW7f8zvwYc0KCEJk8vndc6cZ+Y3v+c338wzyXzzm3nmV5Yxa9asssLCwoMTJ07MmzFjRtcHH3zwo7vuuuvsbdu2pXXv3r1izpw5nW+66abdkydP3v/cc8+dlRhzZfny5Znz5s3rvGrVqvfcnaFDh/YfM2bMJ126dKncvHlz69mzZ28aM2ZM2bZt29LuuOOOvGXLlpX069fvWPJYMxs2bMhYsWJF6b59+1L79+8/4N57793VunVrX7BgQVZhYeH+RL3y8nK78sore+fn5x9+9NFHd2zfvj3t4Ycfzn399dfXd+jQoer+++/v9sMf/jDnoYce2nHLLbf0XrRoUenAgQOPTpgwIW/GjBldH3jggZ0AWVlZFevXr1/31FNPdb7zzjt7LV26dMMjjzySu2jRovW9e/cu3717d627blavXt2ub9+++Tk5OeWPPfbYhwUFBUc+/PDDEwa569mz57GVK1e2q027zbpHpGvbrqy9bS2PX/Y4Y/uMVRIiItJCdevW7VhhYeFBgOuvv37PihUr2qWkpHDttdfuefbZZ7N3796d+tZbb7WbOHHi/urvXbZsWbuxY8fu69ChQ1VWVlbVuHHj9i5durQ9QG5u7rExY8YcDPXaDhs27JN+/fodA8jJyTk+sm1hYeG+Nm3aeG5ubkV2dnb5li1b0gCWLFnS4corrzyQqHfbbbedk0hCEm1+8MEHGcOGDevXr1+//Pnz53fevHlzenFxcUbPnj2PDhw48CjAjTfeuOf3v/99+0Q7U6dO/RjgW9/61serV69uB9GllSlTpuTNnDmzS0VF7Z7+PmLEiIN//vOf3yktLV13++2377z66qvPq1UDn6NZ94iIiEgj8wU9Fw2l+hcOEsu33nrrnnHjxp2XkZHhV1xxxd5WrWo3BEdmZmbVqdRLHk8mNTWViooKC6P5publ5ZUn1hUUFHy6fPnyDocOHfooMzPT3Z2RI0ceePnllzclt/fGG2+0+bztpaR81s9gZg4wb968za+99lrbBQsWZA0dOjR/1apV67p163Y8Wbrzzjt7LF68OAug+gi82dnZx3/Ob37zm/vvvvvus7dv357Wq1evY1u3bj0++N+WLVtO6CE5Fc26R0RERARg+/bt6UuWLGkL8Pzzz2ePGDHiU4C8vLzynJyc8pkzZ+ZOmzbt+KB3aWlpfvToUQMYPXr0pwsXLuwYEoeUhQsXdho9evQn1bcxatSog2+++Wb7kpKSdIDkSzM1+c1vftN+5MiRJ7Rzyy237C4sLNw/fvz4c8vLyxk1atTBoqKidon7PA4cOJDyzjvvtB40aNCRrVu3pifK586d2/niiy8+3tbcuXOzAWbPnt1p8ODBBwHWrl3b+pJLLjn4+OOPb+vUqVPFxo0bTxg9+Mknn9xaUlKyrnoSAtHou1VVUS6ydOnSzKqqKnJyciq++tWvHiwrK8soKSlJP3LkiL344ovZV1999b7P+7mrU4+IiIg0e3l5eUeefPLJs6ZNm5bZp0+fI/fcc8+uxLpJkybtefrpp9OGDBly/JseU6ZM2dW/f//8AQMGHFqwYMGmyZMn7xkyZEh/gOuvv37XRRdddLi0tPSEE3n37t0rnnjiibIJEyacV1VVRefOnctXrFjx/sliWrhwYda11167t3r5gw8++NH06dNTr7rqqt4vvfTSplmzZpVNmjTpS8eOHTOA73//+1sHDhx49JlnnimbOHHiuZWVlQwaNOhQ8s+0d+/e1PPPPz8/PT3d58+fvxFg+vTpPcvKylq7u40cOfLA8OHDD5/q/vvFL37Rac6cOWelpqZ6RkZG1dy5czempKSQkpLCzJkzN1922WXnV1ZWMnny5N0FBQWn/I0ZAHOv19GHG0RBQYEXFRXFHYaISJNiZqvcvSDuOIqLi8sGDRq0+4trNozS0tL08ePH9znZwHc33HDD2YMHDz40ffr0Mxpjfn5+/9WrV5ckX7apDz169PiroqKi93Jzc2t3I0gDKi4u7jJo0KC8mtapR0RERFqsL3/5y/3btGlTNWvWrDN+78q6deua3ci/p0OJiIiINGt9+/Y9drLekLVr1za7ZGDr1q3vxh1DbehmVRERaWhVVVVVGiejhQqf/Um/XaREREREGtqaXbt2ZSkZaXmqqqps165dWcCak9XRpRkREWlQFRUVN+/YseNnO3bsGID+AW5pqoA1FRUVN5+sghIRERFpUEOHDt0JfD3uOKRxUmYqIiIisVEiIiIiIrFRIiIiIiKx0T0iItIyuENlJVRVnfj6RWUN8Z7atHnuuXDNNXHvPZEGo0REpKG5/+UJ5kyd8Br6JBlnm7V9TxMYzqJGV1yhRESaNSUicma4w9GjcOQIHD4cvSamU12uqGiaJ8mmegKsLiUlmlJTP3tNnq+prDbrU1MhPb1+24zzPfXVZpr+TEvzpiO8JUlOBmo66Z9ugnAqdY/UajDGv2QGrVo13B/9RNvN+YRWlzZTUqLPQESknikROdPc4dix+j3J1+a9dWEGbdpEU0bGZ1NiuW1b6NKl5nWnsvx569LSdCIUEWmGWmYikkgGGroX4GR16yKRDJzsxJ2ZCdnZdTvpn6yukgEREalnzTsRWb8eJkyo+TJBXa7bJycDNZ24k5OB+uwVyMiILiEoGRARkWaieScimZmQn396CcHnJQhKBkREROpF805EevaEF16IOwoRERE5CT1ZVURERGKjRERERERio0REREREYqNERERERGKjRERERERiE1siYmaXmVmpmW0ws+/FFYeIiIjEJ5ZExMxSgaeBy4F84Dozy48jFhEREYlPXD0iw4AN7r7R3Y8B84FvxBSLiIiIxCSuB5r1AD5MWt4C/HVyBTObBkwLi5+aWWkdttcF2F2H9zcUxVU7iqt2FFftNMe4zqnPQEQaQqN9sqq7/xT4aX20ZWZF7l5QH23VJ8VVO4qrdhRX7SgukXjEdWlmK9ArablnKBMREZEWJK5E5E9AHzPrbWbpwCRgQUyxiIiISExiuTTj7hVmdgfwKpAKzHH3tQ24yXq5xNMAFFftKK7aUVy1o7hEYmDuHncMIiIi0kLpyaoiIiISGyUiIiIiEpsmmYiYWS8zW2pm68xsrZndFcqzzWyxmb0fXjuFcjOzJ8Lj5N8xsyFJbU0N9d83s6l1jCvDzN40s+IQ1w9CeW8zWxm2/6twgy5m1josbwjr85Laui+Ul5rZ1+oSV1KbqWa22sxeaSxxmVmZmb1rZm+bWVEoi/VzDO11NLNfm1mJmb1nZhfGHZeZ9Q37KTEdMLNvxx1XaG96OObXmNkvw+9CYzi+7goxrTWzb4eyM76/zGyOme00szVJZfUWh5kNDb9HG8J77fT2mEgM3L3JTUAuMCTMtwfWEz0q/sfA90L594BHw/xY4LeAAcOBlaE8G9gYXjuF+U51iMuAdmG+FbAybO+/gEmh/Bng1jB/G/BMmJ8E/CrM5wPFQGugN/ABkFoP++1uYB7wSliOPS6gDOhSrSzWzzG0+R/AzWE+HejYGOJKii8V2EH0wKq4j/sewCagTdJxdWPcxxcwAFgDZBLdmL8EOC+O/QV8BRgCrGmI4xx4M9S18N7L6+M406TpTEyxB1AvPwT8L/A3QCmQG8pygdIwPwu4Lql+aVh/HTArqfyEenWMKRN4i+iJsbuBtFB+IfBqmH8VuDDMp4V6BtwH3JfU1vF6dYinJ/A74BLglbCdxhBXGX+ZiMT6OQJZRCdWa0xxVYulEPhDY4iLz56UnB2Ol1eAr8V9fAETgdlJy/8C/GNc+wvI48REpF7iCOtKkspPqKdJU2OfmuSlmWShW3cwUe9DjrtvD6t2ADlhvqZHyvf4nPK6xJNqZm8DO4HFRP/V7XP3ihq2cXz7Yf1+oHNDxAU8TvRHuCosd24kcTmwyMxWWfRYf4j/c+wN7AJ+btGlrJ+ZWdtGEFeyScAvw3yscbn7VuDfgM3AdqLjZRXxH19rgIvNrLOZZRL1NPSi8XyO9RVHjzBf3/GJnBFNOhExs3bAfwPfdvcDyevc3YlOcmeUu1e6+wVEPRDDgH5nOobqzGw8sNPdV8UdSw1GuvsQopGYbzezrySvjOlzTCPqRv93dx8MHCTqOo87LgDCvRZfB16ovi6OuMK9Dd8gSuC6A22By85kDDVx9/eAR4FFwP8BbwOV1erE9jk2xjhE4tBkExEza0WUhDzv7i+G4o/MLDeszyXqlYCTP1K+wR417+77gKVEXdIdzSzx8LjkbRzfflifBexpgLguAr5uZmVEIx1fAvykEcSV+G8ad98J/A9R8hb357gF2OLuK8Pyr4kSk7jjSrgceMvdPwrLccd1KbDJ3Xe5eznwItEx1xiOr9nuPtTdvwLsJbqfLO79lVBfcWwN8/Udn8gZ0SQTkXBH+GzgPXd/LGnVAiBxJ/lUontHEuU3hLvRhwP7Q5foq0ChmXUK/9UVhrLTjaurmXUM822I7lt5jyghueYkcSXivQZ4LfxntACYFL5d0BvoQ3Qz2mlx9/vcvae75xF16b/m7lPijsvM2ppZ+8Q80f5fQ8yfo7vvAD40s76haAywLu64klzHZ5dlEtuPM67NwHAzywy/m4n9FevxBWBmZ4XXs4GriG7Wjnt/JdRLHGHdATMbHvb/DUltiTR+cd+kcjoTMJKoG/Mdou7Wt4mu/3YmuiHzfaI75LNDfQOeJrpf412gIKmtvwM2hOmmOsY1EFgd4loDPBDKv0T0B3UDUXd661CeEZY3hPVfSmrr/hBvKfV4Bzwwis++NRNrXGH7xWFaC9wfymP9HEN7FwBF4bN8iehbCo0hrrZEvQdZSWWNIa4fACXhuP9Pom++xH7cA8uJkqJiYExc+4socdwOlBP1uP19fcYBFIR9/wHwFNVutNakqTFPesS7iIiIxKZJXpoRERGR5kGJiIiIiMRGiYiIiIjERomIiIiIxEaJiIiIiMRGiYg0a2ZWadFItcVm9paZjfiC+h3N7LZTaHeZmRWcZkwLE8+bERFp6ZSISHN32N0vcPdBRIOq/esX1O9INDpsg3H3sR49eVdEpMVTIiItSQeix3xjZu3M7Hehl+RdM/tGqPMIcG7oRZkR6n431Ck2s0eS2ptoZm+a2Xozu7j6xsws18xeD22tSdQxszIz62Jm/xDWvW1mm8xsaVhfaGZvhNhesGhMJRGRZkkPNJNmzcwqiZ5OmUE0XPol7r4qjHGS6e4HzKwL8EeiR4qfQ/Tk2QHh/ZcTDR9/qbsfMrNsd//YzJYBq9z9O2Y2Frjb3S+ttu3vABnu/iMzSw3b+8SiMX8K3H13qNcKeA34MfAG0Vgtl7v7QTP7LtETSR9qyP0kIhKXtC+uItKkHfZoNGTM7EJgrpkNIHqM9sMWjfZbRTRsek4N778U+Lm7HwJw94+T1iUGW1wF5NXw3j8Bc0Ki8ZK7v32SGH9CNN7KyxaNlJwP/CEaNoR0ouRERKRZUiIiLYa7vxF6P7oSjU3UFRjq7uWhlyKjlk0eDa+V1PC75O6vh0RnHPCcmT3m7nOT65jZjUS9MHckioDF7n5dLWMREWmSdI+ItBhm1g9IJQwaB+wMSchoomQA4BOgfdLbFgM3mVlmaCO7Fts7B/jI3Z8FfgYMqbZ+KHAP8LfuXhWK/whcZGbnhTptzez82v2kIiJNh3pEpLlrY2aJSyIGTHX3SjN7HnjZzN4lGmW3BMDd95jZH8xsDfBbd7/XzC4AiszsGLAQ+KdT3PYo4F4zKwc+JRqePdkdQDawNFyGKXL3m0MvyS/NrHWo98/A+lr/5CIiTYBuVhUREZHY6NKMiIiIxEaJiIiIiMRGiYiIiIjERomIiIiIxEaJiIiIiMRGiYiIiIjERomIiIiIxOb/ASTynngOYF91AAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] @@ -386,17 +367,12 @@ } ], "source": [ - "plot_relative_time(results, n_features, max_batch_size=max_batch_size)" + "plot_relative_time(experiments_eq, results, n_features, max_batch_size=max_batch_size)" ] }, { "cell_type": "markdown", - "id": "b96a904b", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ "The difference between KeOps and PyTorch is even more striking when we only look at $[2, 10]$ features:" ] @@ -404,16 +380,11 @@ { "cell_type": "code", "execution_count": 7, - "id": "0d1e4dfa", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAEWCAYAAAAQHy/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdeVhV1frA8e9iEFGBRJFwQBxAUbQcU8uRQXPMzCEShyyzbllqdbv3dpt/TYqVXsuh1BwrLcspp0TFzJxnMSccUVFEQGU86/fHPigSICCwGd7P85xHzt777PXuc45nvXvttddSWmuEEEIIUbbYmB2AEEIIIYqeJABCCCFEGSQJgBBCCFEGSQIghBBClEGSAAghhBBlkCQAQgghRBkkCUAJoZTSSqn6OayfqpT6b4bnzyulLiqlEpRSVfJRnpe1TLv8xlxQrMdQtwjK8bSWZVsEZbkrpTYppeKVUqGFXZ7IPaXUv5VSX5sdhxCFTRKATJRSkUqpZKVU1UzLd1srRC/r89nW530ybfeZdfkw6/NhSqk0a8WSoJQ6qZSapZTyKci4tdajtNbvW8u0ByYCQVrrSlrrK3dLIO6F9Rg3F9C+Niilnsm4zHoMJwpi/5nKilRKBWQo57S1rLSCLisLI4HLgLPWety97kwp5aGUWqqUOp/xe5phvYNSaqZSKk4pdUEpNTbTen+lVIRS6oZSKkwpVfteYzKDUuodpdS8PGzfSSl1NuMyrfWHWutnsnuNEKWFJABZOwk8mf5EKdUEqJDFdn8BQzJsZwcMAI5n2u4PrXUlwAUIAG4CO5VSfgURbBZnrO5AeeBgQexfFIrawCGdj5G4smmVsQCrgH7ZvOwdwNtabmfgdaVUN+v+qgI/Af8FXIEdwPd5jSuvikPrkhBlmtZaHhkeQCTwJrA9w7IJwH8ADXhZl822Lr8IVLYu6wn8CmwGhlmXDQM2Z1HOcmBxDnG8BkQB54GnrWXXz1D2V8BK4DpGUjEb+ADwsS7TQAKwHthkfX7dumxgFuXZWo/nMnAC+If1NXbW9S7AN9aYzlnLsgV8gUQgzbrvWOv2Dtb9nba+R1MBxwzl9QH2AHEYCVM34P+s+0m07ut/1m0zHrsLMAeIBk5ZPyubjO+1tdyrGInco9m8v3MxKs2b1rJeB7wyHfMG63FusW6zDKgCzLfGvT39+2DdviGwFogBjgADsil7NpACJFv3G2B9vz63ft7nrX87WLfvBJwF/glcAObm8L2xI8P3NMPy8xgtQunP3we+s/49EtiSYV1F6/vSMIf/I/8CDlnf51lA+Qzre1o/21jre9c002v/CewDkqzxRmJ83/dhfEe/wUhifwXigXXc/j/WCTibRTwBGN+hZOt7mwDsta4fDhy27usE8Fym47RYt08AqmMkS/My7L83RjIda/1O+GYq+1Vr7NcwEqfy1nVVMf6fx1q/E+FYv6vykEdxeJgeQHF7ZPgxOYJRudlaf3xr8/cE4ANgOvC8ddkPGC0HuUkAngYuZhNDN4xK08/6I7WAvycA14CHMVpxyqfHY13vRYaKzLrs1uuzKXMUEAHUwjgLDOPOynAJMM0aTzVgW4Yf0r8dI/AZsNS6LyeMyvMj67rW1vgDrfHXwFrZWH9gn8m0r4zHPgf4xbpPL4xWmBEZ4kgBnrV+bs9jVHwqp886w/M73jdrLMeAehiJxyFreQEYFdccYJZ124rAGYzKxg5ohpFMNcqm7Fufl/X5e8BW63vrhlFxvm9d1wlIBT7BSBQcs9qnddu/JQBAZesy9wzLngD2W//+Avgq034OAP1yeN8OZPiu/M7t714z4BLwkPUzGGrd3iHDa/dYX+uYYdlWjEq/hvX1u6z7Ko+RxL6d4b3IMgGw/v0OGSpv67Ie1s9QAR2BG0DzHPZ3ax/cTqgDAXuMRPEYUC5D2dswEgdXjERjlHXdRxiJr7310Z5svovykIcZD7kEkL25GM37gRj/qc9ls90cYIhS6j6MH5efc7n/8xg/GFkZgFGxHNBaX8f4QcrsF63171pri9Y6MZdl5mQA8LnW+ozWOgbjxwswOqwB3YFXtNbXtdaXMCr4QVntSCmlMM4qx2itY7TW8cCHGbYfAczUWq+1xn9Oax1xtwCtlzoGAf/SWsdrrSOBUCAkw2antNYztHEd/1vAA6Niya9ZWuvjWutrGGekx7XW67TWqcAijEoKjLPeSK31LK11qtZ6N/Aj0D+X5TwFvKe1vqS1jgbezXRcFoxKMElrfTOPx1DJ+u+1DMuuYSRR6euvcaeM67Pyvwzflf/j9iWzkcA0rfWfWus0rfW3GGf6bTK8dpL1tRmPY7LW+qLW+hzGmfKfWuvd1u/2Em6/z3mmtV5h/Qy11nojsAajMs6NgcAK63c1BaN1yRFol+l4zlvfi2XAg9blKRjfv9pa6xStdbjWWiZfEcWGXIPL3lyMpvM6GJV8lrTWm5VSbhiXCJZrrW8a9d9d1cBoFsxKdWBnhuenstjmTG4KyYPqmfaZsczaGGcwURmOzSaHGNww+kzszLC9wjgjBOPsb2U+YqxqjSNjbKcw3st0F9L/0FrfsJZfify7mOHvm1k8T993beAhpVRshvV2GN+j3KjO34+reobn0feQ6CVY/3XGuLyS/nd8hvXOmV6TcX1WMn9X0mOtDQxVSr2UYX057jyWrL43uX2f80wp9SjwNsbZvA3Gd3N/Ll9+x+eitbYopc6QzXcOo3Uh/VjHYyTva6zfw+la64/zcQhCFAppAciG1voUxjXk7hgdpHIyDxhHDolCFvpinOlkJQqjkkznmVWIeSgrN3Iq8wzGWVxVrfV91oez1rpxNrFcxvjRbpxhexdtdIRM31+9bOLI6bguY5xVZeyh7kn2rTN3U5Dv4RlgY4bjvU8bdxQ8n8vXn+fvx3U+w/N8x6q1vorx+T6QYfED3O4kejDjOqVURYzPJ6dOpJm/K+mxngH+L9P7UEFrvTBjSPk7EsBojr/VIdfaKuSW3b6VUg4YLTETMC6B3IeRfKqsts/CHZ+LtXWrFrn4zllbqcZpreti9CMYq5Tyv9vrhCgqkgDkbATQxdoMn5NJGJcKNuW0kVLKVilVRyk1GePa47vZbPoDMEwp1UgpVQHj7OVeXQRyupf+B2C0UqqmUqoy8Eb6Cq11FEazaahSylkpZaOUqqeU6phh3zWVUuWs21uAGcBnSqlqAEqpGkqprtbtvwGGW289s7Gua3i3OK3N+j8A/6eUcrLeqjYWIwHLj7u9J3mxHPBRSoUopeytj1ZKKd9cvn4h8KZSys3aK/8t8nhcSqnyGH0EABysz9PNse6/svW9fhajHwIYTex+Sql+1te8Bey7y2WZf1i/K64YrV/pdw3MAEYppR5ShopKqR5KqZwuJ+TFX0B56z7tMTqBOmRYfxHwUkql/7aVs66PBlKtrQFBmbavopRyyaa8H4Ae1u+qPUain4TRRyNHSqmeSqn61qThGkYHV0tuD1SIwiYJQA6s1w135GK7GK31bzlc32urlErA6Dm+AaN5tZXWOstmSK31rxi9wNdjdDhan5/4M3kH+FYpFauUGpDF+hnAamAvRgeszK0eQzB+TNN7fi/GuL6JNb6DwAWl1GXrsn9aY9+qlIrD6MndwHp82zA6y32G8cO4kdtnWV8ATyilriqlJmUR50sYZ4EnMDpbLgBm5u4t+JuPMCrFWKXUq/ncB2Cc7WFULIMwzhovcLvTXm58gHH73T6M5uld1mV5kX5HAxgdOjNeY38b426LUxjv93it9Spr7NEYtw/+H8Zn+xDZ9O/IYAFGUnjCut8PrPvagZFc/M+6r2MYnTMLhLUvxgvA1xhn4dcxOummW2T994pSapf1cxmNUZFfBYIxOqem7y8CI/k6Yf0eZLxUgdb6CDAYmIzRAtUL6KW1Ts5FuN4Y3/sE4A/gS611WN6OWIjCo6RPihAiL5RSkRh3aqwzOxYhRP5JC4AQQghRBkkCIIQQQpRBcglACCGEKIOkBUAIIYQog0rEQEBVq1bVXl5eZochhBAlys6dOy9rrd3uvqUoi0pEAuDl5cWOHXe9G08IIUQGSqmsRhEVApBLAEIIIUSZVGgJgFJqplLqklLqQBbrximltHXEMyGEEEIUscJsAZiNMa3tHZRStTBGTDtdiGULIYQQIgeFlgBorTeR9Wx3n2HMqS33HwohhBAmKdI+AEqpPsA5rfXeoixXCCGEEHcqsrsArLPa/Zs7Z+LKafuRwEgAT8+sZsMVQgghRH4VZQtAPaAOsNc6mUhNYJdS6v6sNtZaT9dat9Rat3Rzk9tYhRBCiIJUZAmA1nq/1rqa1tpLa+2FMYVnc631haKKQQghSorrydd5+deXiU2MNTsUUUoV5m2ACzHmwG6glDqrlBpRWGUJIURpEpsYS9C8IP63/X/8fvp3s8MRpVSh9QHQWj95l/VehVW2EEKUVNHXowmaF8TBSwf54Ykf6OHTw+yQRClVIoYCFkKIsuBc3DkC5gYQGRvJ0ieX0q3+34ZSEaLASAIghBDFwImrJwiYE8DlG5dZPXg1HWp3MDskUcpJAiCEECY7FH2IwLmBJKYm8tuQ32hVo5XZIYkyQBIAIYQw0a6oXXSd1xVbZcvGYRvxq+ZndkiijJDZAIUQwiS/n/6dzt92poJ9BcKHh0vlL4qUJABCCGGCdSfWETQviPsr3c/m4ZvxruJtdkiijJEEQAghitgvEb/QY0EP6lWux6Zhm6jlUsvskEQZJAmAEEIUoQX7F9Dvh348eP+DbBi2AfdK7maHJMooSQCEEKKITN85ncE/DaZ97fasC1mHq6Or2SGJMkwSACGEKAKhW0J5bvlzPOr9KCuDV+Lk4GR2SKKMkwRACCEKkdaat8Pe5tW1r9K/UX+WDFyCo72j2WEJIeMACCFEYdFaM27NOD7b+hnDHxzOjF4zsLWxNTssIQBJAIQQolCkWdIYtXwUX+/+mtGtR/NZt8+wUdLoKooPSQCEEKKApaSlMOTnIXx34Dv+0/4/vN/5fZRSZoclxB0kARBCiAKUmJrIgEUDWPbXMj72/5h/PvJPs0MSIkuSAAghRAFJSE7gse8e47eTvzGl+xReaPWC2SEJkS1JAIQQogDEJsbSfX53/jz3J3Mem0PIAyFmhyREjiQBEEKIe3Tp+iW6zuvKwUsHWdR/EY/7Pm52SELclSQAQghxD87GnSVwbiCnYk+x7MlldK3f1eyQhMgVSQCEECKfTlw9gf8cf67cuMLqwatpX7u92SEJkWuFdlOqUmqmUuqSUupAhmXjlVIRSql9SqklSqn7Cqt8IYQoTIeiD/HIzEeIS4pj/dD1UvmLEqcwR6WYDXTLtGwt4Ke1bgr8BfyrEMsXQohCsStqFx1mdUCj2ThsIy2rtzQ7JCHyrNASAK31JiAm07I1WutU69OtQM3CKl8IIQrD76d/p/O3nalUrhLhw8Pxq+ZndkhC5IuZ41I+Dfya3Uql1Eil1A6l1I7o6OgiDEsIIbK29vhaguYFcX+l+wkfHk591/pmhyREvpmSACil/gOkAvOz20ZrPV1r3VJr3dLNza3oghNCiCz8HPEzPRf2pL5rfTYN20Qtl1pmhyTEPSnyBEApNQzoCTyltdZFXb4QQuTV/H3zeeKHJ2h2fzPChobhXsnd7JCEuGdFmgAopboBrwO9tdY3irJsIYTIj2k7phGyJIQOtTuwNmQtro6uZockRIEozNsAFwJ/AA2UUmeVUiOA/wFOwFql1B6l1NTCKl8IIe7VhC0TGLViFN29u7MieAVODk5mhyREgSm0gYC01k9msfibwipPCCEKitaatze8zfub3mdA4wHM7TuXcrblzA5LiAIlIwEKIUQGWmvGrh7L539+ztMPPs30XtOxtbE1OywhCpwkAEIIYZVmSWPU8lF8vftrXn7oZSZ2nYiNMvNuaSEKjyQAQggBpKSlELIkhO8Pfs+b7d/kvc7voZQyOywhCo0kAEKIMi8xNZH+i/qz/K/lfBLwCa8//LrZIQlR6CQBEEKUaQnJCfT5rg9hJ8P4svuXPN/qebNDEqJISAIghCizrt68SvcF3dl+bjtz+s5hcNPBZockRJGRBEAIUSZdun6JoLlBHIo+xKL+i+jr29fskIQoUpIACCHKnLNxZwmYE8Dpa6dZ9uQyutbvanZIQhQ5SQCEEGXK8Zjj+M/x52riVdaErOERz0fMDkkIU0gCIIQoMw5eOkjg3ECS05JZP2Q9Laq3MDskIUwjI1wIIcqEned30nF2RzSajcM2SuUvyjxJAIQQpd7m05vpMqcLlcpVInx4OI2rNTY7JCFMJwmAEKJUW3N8DUFzg/Co5MHmpzdT37W+2SEJUSxIAiCEKLWWHF5Cr4W98Kniw6bhm6jpXNPskIQoNiQBEEKUSvP2zaP/ov4092hO2NAwqlWsZnZIQhQrkgAIIUqdqTumMmTJEDp6dWRtyFoqO1Y2OyQhih1JAIQQpcr438fz/Irn6eHTgxXBK6hUrpLZIQlRLEkCIIQoFbTW/Hf9f3l93esMbDyQnwb8RHm78maHJUSxJQMBCSFKPK01Y1aP4Ys/v2BEsxFM6zkNWxtbs8MSoliTBEAIUaKlWdJ4bvlzfLP7G1556BUmdp2IUsrssIQo9grtEoBSaqZS6pJS6kCGZa5KqbVKqaPWf6VnjhAi35LTkgn+KZhvdn/Dfzv8Vyp/IfKgMPsAzAa6ZVr2BvCb1tob+M36XAgh8uxmyk0e//5xfjj4A58GfMp7nd+Tyl+IPCi0BEBrvQmIybS4D/Ct9e9vgccKq3whROmVkJxAjwU9WHl0JVN7TOW1h18zOyQhSpyi7gPgrrWOsv59AXDPbkOl1EhgJICnp2cRhCaEKAmu3rxK9wXd2X5uO3P6zmFw08FmhyREiWTabYBaaw3oHNZP11q31Fq3dHNzK8LIhBDF1cWEi3T6thO7onaxqP8iqfyFuAdF3QJwUSnlobWOUkp5AJeKuHwhRAl15toZAuYGcObaGZY9uYygekFmhyREiVbULQBLgaHWv4cCvxRx+UKIEuhYzDHaz2rPhYQLrAlZI5W/EAWg0FoAlFILgU5AVaXUWeBt4GPgB6XUCOAUMKCwyhdClA4HLh0gcG4gKWkprB+ynhbVW5gdkhClQqElAFrrJ7NZ5V9YZQohSpcd53fQdV5XHGwd2DR8E43cGpkdkhClhswFIIQolsJPhdPl2y44Oziz+enNUvkLUcAkARBCFDurj62m67yuVHeqTvjwcOpWrmt2SEKUOpIACCGKlZ8O/0Svhb3wqeLDpuGbqOlc0+yQhCiVJAEQQhQbc/fOZcCiAbSs3pKwoWFUq1jN7JCEKLUkARBCFAtfbf+KIT8PoaNXR9aErKGyo8wVJkRhkgRACGG6T3//lBdWvkAvn16sCF5BpXKVzA5JiFJPEgAhhGm01ry5/k3+ue6fDGw8kB8H/Eh5u/JmhyVEmVDUQwELIQQAFm1hzKoxTNo2iWeaPcPUnlOxtbE1OywhygxJAIQQRS7NksbIZSOZuWcmY9qMITQoFKWU2WEJUaZIAiCEKFLJackM/mkwiw4t4q0Ob/FOp3ek8hfCBJIACCGKzM2Um/Rf1J8VR1cwPnA8r7Z71eyQhCizJAEQQhSJ+KR4en/Xm42RG5nWcxojW4w0OyQhyjRJAIQQhS7mZgyPzn+Uned3MrfvXJ5q+pTZIQlR5kkCIIQoVBcTLhI0L4iIyxEsHrCYxxo+ZnZIQggkARBCFKIz184QMDeAs3FnWf7kcgLrBZodkhDCShIAIUShOBZzDP85/sQmxrJm8Boe9nzY7JCEEBlIAiCEKHAHLh0gcG4gKWkphA0No7lHc7NDEkJkIkMBCyEK1I7zO+g4uyM2yoZNwzdJ5S9EMSUJgBCiwGw6tYku33bB2cGZ8OHhNHJrZHZIQohsSAIghCgQq46totu8btRwrsHm4ZupW7mu2SEJIXJgSgKglBqjlDqolDqglFqolJLpv4QowX489CO9F/amQdUGbBy2kRrONcwOSQhxF3dNAJRSbZVSU5RS+5RS0Uqp00qplUqpfyilXPJaoFKqBjAaaKm19gNsgUF5D10IURzM2TuHAYsH0LJ6S8KGhlGtYjWzQxJC5EKOCYBS6lfgGWA10A3wABoBbwLlgV+UUr3zUa4d4KiUsgMqAOfzsQ8hhMm+3P4lQ38eSmevzqwJWcN95e8zOyQhRC7d7TbAEK315UzLEoBd1keoUqpqXgrUWp9TSk0ATgM3gTVa6zWZt1NKjQRGAnh6eualCCFEEfhk8ye88dsb9PLpxQ/9f6C8nVzJE6IkybEFIL3yV0pVVErZWP/2UUr1VkrZZ9wmt5RSlYE+QB2gOlBRKTU4i7Kna61baq1burm55aUIIUQh0lrzn9/+wxu/vcGTfk/y44AfpfIXogTKbSfATUB56/X7NUAIMDufZQYAJ7XW0VrrFOAnoF0+9yWEKEIWbeHlVS/z4eYPebb5s8ztOxd7W3uzwxJC5ENuEwCltb4BPA58qbXuDzTOZ5mngTZKqQpKKQX4A4fzuS8hRBFJs6QxYukIJm+bzNg2Y5nWcxq2NrZmhyWEyKdcJwBKqbbAU8AK67J8/c/XWv8JLMboQ7DfGsP0/OxLCFE0ktOSefLHJ5m9ZzZvd3ybCUETMPJ3IURJldu5AF4G/gUs0VofVErVBcLyW6jW+m3g7fy+XghRdG6m3OSJRU+w8uhKJgROYFy7cWaHJIQoALlKALTWmzD6AaQ/P4FxL78QohSLT4qn18JebDq1iWk9pzGyxUizQxJCFJC7jQMwQynVJJt1FZVSTyulniqc0IQQZoq5GUPA3AA2n97MvMfnSeUvRClztxaAKcB/rUnAASAaYwAgb8AZmAnML9QIhRBF7mLCRQLnBnLkyhF+HPAjfRr2MTskIUQByzEB0FrvAQYopSoBLTFGArwJHNZaHymC+IQQRez0tdMEzAngXPw5VgSvIKBugNkhCSEKQW77ACQAGwo3FCGE2Y5eOUrA3ACuJV5jbcha2tWSITqEKK1yexeAEKKU239xP4FzA0nTaawfup7mHs3NDkkIUYhMmQ5YCFG8bD+3nU7fdsLWxpZNwzZJ5S9EGZCnBEApVaGwAhFCmGPTqU34z/HHxcGF8OHh+Lr5mh2SEKII5CoBUEq1U0odAiKszx9QSn1ZqJEJIQrdqmOr6DqvKzWcaxA+PJy6leuaHZIQoojktgXgM6ArcAVAa70X6FBYQQkhCt+Ph36k98Le+Fb1ZdOwTdRwrmF2SEKIIpTrSwBa6zOZFqUVcCxCiCLy7Z5vGbB4AK1qtGL90PW4VZQpt4Uoa3KbAJxRSrUDtFLKXin1KjKDnxAl0pRtUxj2yzA6e3VmzeA13Ff+PrNDEkKYILcJwCjgH0AN4BzwoPW5EKIE+Xjzx7z464v0btCb5cHLqViuotkhCSFMktuBgC5jTAUshCiBtNb8Z/1/+GjzRwQ3CWZ2n9nY29qbHZYQwkS5SgCUUnWAlwCvjK/RWvcunLCEEAXFoi28/OvL/G/7/xjZfCRf9vgSWxtbs8MSQpgstyMB/gx8AywDLIUXjhCiIKVaUnl22bPM3jObcW3HMT5wPEops8MSQhQDuU0AErXWkwo1EiFEgUpOS+apn55i8aHFvNPxHd7q+JZU/kKIW3KbAHyhlHobWAMkpS/UWu8qlKiEEPfkRsoNnvjhCX499iuhQaGMbTvW7JCEEMVMbhOAJkAI0IXblwC09bkQohiJS4qj18JehJ8KZ3rP6Tzb4lmzQxJCFEO5TQD6A3W11smFGYwQ4t7E3Iyh27xu7IraxfzH5/NkkyfNDkkIUUzldhyAA0CBjRailLpPKbVYKRWhlDqslGpbUPsWoqy6kHCBjrM7su/iPn4a+JNU/kKIHOW2BeA+IEIptZ07+wDk9zbAL4BVWusnlFLlAJllUIh7cPraaQLmBHAu/hwrglfgX9ff7JCEEMVcbhOAtwuqQKWUC8ZEQsMArJcV5NKCEPl09MpR/Of4E5cUx9qQtbSr1c7skIQQJUBuRwLcWIBl1gGigVlKqQeAncDLWuvrGTdSSo0ERgJ4enoWYPFClB77L+4ncG4gaTqNsKFhNPNoZnZIQogSIsc+AEqpzdZ/45VScRke8UqpuHyWaQc0B77SWjcDrgNvZN5Iaz1da91Sa93SzU1mKhMis23nttFxdkfsbOzYNGyTVP5CiDy5WyfAigBaayettXOGh5PW2jmfZZ4Fzmqt/7Q+X4yREAghcmlj5Eb85/hzX/n7CB8ejq+br9khCSFKmLslALqgC9RaX8CYXriBdZE/cKigyxGitPr16K90m9+NWs61CB8eTp3KdcwOSQhRAt2tD0A1pVS2Q4hprSfms9yXgPnWOwBOAMPzuR8hypTFhxYT/GMwftX8WD14NW4V5fKYECJ/7pYA2AKVgAIdQFxrvQdoWZD7FKK0m71nNiOWjqBtzbasCF6BS3kXs0MSQpRgd0sAorTW7xVJJEKIbP1v2/946deXCKgbwM8Df6ZiuYpmhySEKOHu1gdApg4TwmQfhX/ES7++RJ8GfVj25DKp/IUQBeJuLQAynJgQJtFa8+/f/s3Hv39McJNgZveZjb2tvdlhCSFKiRwTAK11TFEFIoS4zaItjP51NFO2T+G5Fs/xZY8vsVG5nbpDCCHuLrdDAQshikiqJZVnlj7Dt3u/ZVzbcYwPHI9ScjVOCFGwJAEQohhJTksm+Mdgfjz8I+91eo83O7wplb8o0Xbu3FnNzs7ua8CP3M9AKwqGBTiQmpr6TIsWLS5lXikJgBDFxI2UG/T7oR+rjq3is66f8UqbV8wOSYh7Zmdn9/X999/v6+bmdtXGxqbAB5cT2bNYLCo6OrrRhQsXvgb+NnuvZGNCFANxSXE8Ov9RVh9bzYxeM6TyF6WJn5ubW5xU/kXPxsZGu7m5XcNoffkbaQEQwmRXblzh0fmPsvvCbhb0W8Agv0FmhyREQbKRyt881vc+y5N9SQCEMNGFhAsEzg3k6JWj/DTgJ3o16GV2SEKIMkIuAQhhklOxp2g/qz0nr55kRfAKqfyFKARHjhwp5+3t3djsOLLTu3fvOhNP5UYAACAASURBVF5eXn7e3t6N+/fv75WUlFRkvX4lARDCBH9d+Yv2s9oTfT2atSFr8a8rY24JURY99dRTMSdOnDhw5MiRg4mJierzzz+vWlRlSwIgRBHbd3EfHWZ1IDE1kQ3DNtC2VluzQxKiTDh06FA5X1/fRhs3bqyQmprKc889V9PPz8/Xx8en0fjx46sCWCwWnnvuuZre3t6NfXx8Gs2YMaMywPLly51atmzZoFOnTvW9vLz8goODPdPS0khNTaVfv35e6du/++671fIS08CBA6/Z2NhgY2NDy5Ytr589e7ZcYRx7VqQPgBBF6M+zf/Lo/EepYF+BdUPW0bBqQ7NDEqLIPP00tQ4coEJB7tPPjxszZ3Lmbtvt3bvXYdCgQfVmzpx5sm3btjcnTJhQ1cXFJe3AgQOHb968qVq1atWwV69ecVu3bq2wf/9+x8OHDx+Mioqya926tW9QUFACwP79+yvu3r37gI+PT3KHDh2858yZU7l+/fpJUVFR9kePHj0IcPnyZdv8HEdSUpL6/vvvq0ycOPGux1JQpAVAiCKyIXIDAXMDqOxYmfDh4VL5C1FEYmJi7B577LH68+bNO9G2bdubAOvWrXP+4YcfqjRs2LBRs2bNfK9evWp36NCh8uHh4U4DBgyIsbOzo1atWqkPPfRQwubNmysANGnS5HqjRo2S7ezsGDBgQEx4eHilhg0bJp05c8Zh6NChtRYvXuxcuXLltPzEOHToUM82bdokdOvWLaEgjz0n0gIgRBFYeXQl/X7oR93KdVkbspbqTtXNDkmIIpebM/XC4OTklFa9evXksLCwSi1atEgE0Fqr0NDQ0/369YvLuO2KFStcsttP5lE5lVK4ubmlHThw4NCSJUucp06d6vb999+7Llq0KDJ9m9TUVPz8/BoBdOvWLfbzzz8/n3m/48aN87h8+bLd6tWrj9/bkeaNtAAIUcgWHVzEY989RiO3RmwctlEqfyGKmL29vf7111+PL1y4sMrUqVNdAQIDA6999dVXbum97vft2+cQFxdn06FDh/jFixe7pqamcv78ebtt27ZVat++/XUwLgFERESUS0tLY/Hixa7t27ePj4qKsktLS2PYsGGxH3300bn9+/ffcYnDzs6OiIiIQxEREYeyqvwnTpxYdf369S4///zzCVvbfF09yDdpARCiEM3aPYtnlj1D25ptWRG8Apfy2Z5cCCEKkbOzs2X16tXHOnXq5OPk5JQ2ZsyYy5GRkQ5NmjTx1VorV1fXlJUrVx4PCQmJ3bJlSyVfX9/GSin97rvvnvX09Ezdt28ffn5+10eNGuUZGRlZvl27dnEhISGx27ZtcxwxYoSXxWJRAO+9997ZvMT1+uuv1/bw8Ehq2bKlL0DPnj2vTpgwIaow3oPMlNbFf4Cmli1b6h07dpgdhhB5MvnPyYxeNZrAuoEsGbiEiuUqmh2SKGOUUju11i3NjGHv3r2RDzzwwGUzYygIy5cvdwoNDXUPCws7ZnYsebV3796qDzzwgFfm5aZdAlBK2SqldiullpsVgxCF5cPwDxm9ajSPNXyMZU8uk8pf5NnNm7B6NcTHmx2JKK3M7APwMnDYxPKFKHBaa95Y9wb/Wf8fBjcdzKL+i3CwczA7LFECaA1//QWTJkH37lClCnTrBr/9ZnZkAqBnz57xJfHsPyem9AFQStUEegD/B4w1IwYhCppFW3hp5Ut8ueNLRrUYxZQeU7BR0s9WZC8hAcLCYNUq+PVXOHnSWN6gAYwcaSQAHTuaG6MovczqBPg58DrgZFL5QhSoVEsqI5aOYM7eObza9lU+Dfz0b7cMCaE1HDx4u8IPD4eUFKhYEfz94bXXjEq/Th2zIxVlQZEnAEqpnsAlrfVOpVSnHLYbCYwE8PT0LKLohMi7pNQkgn8K5qfDP/F+5/f5T/v/SOUvbomNhXXrjEp/1So4d85Y3qQJvPIKPPooPPwwlCuyAWCFMJjRAvAw0Fsp1R0oDzgrpeZprQdn3EhrPR2YDsZdAEUfphB3dyPlBo9//zirj6/ms66f8UqbV8wOSZjMYoHdu29X+H/8AWlp4OICgYFGhd+1K9SoYXakoqwr8guUWut/aa1raq29gEHA+syVvxAlQVxSHN3mdWPN8TV83etrqfzLsMuXYcECGDIEPDygZUt4802jJ/+//gWbNxvbLFoETz8tlX9x98Ybb9xfUPsaO3Zs9bfeess9v6/fsmWL44MPPtiwfv36d0xOVBBkICAh8uHKjSt0m9+NPRf2sLDfQgb6DTQ7JFGE0tJg27bbZ/nbtxvX96tWNc7uu3WDoCColqd54URxMWnSJI+PP/74Qm63t1gsaK0pjJH8KlWqZJk7d+7JJk2aJEVGRtq3atXKt2/fvnFVq1bN15wDGZnaRVlrvUFr3dPMGITIq6j4KDrO7sj+i/tZMnCJVP5lRFQUzJ4NgwaBmxu0awcffAB2dvDuu0ZCcOECzJsHgwdL5V9cHDlypFydOnUa9+7du07dunUbd+vWrW58fLzN0qVLnQICAuqlb7dkyRLnwMDAei+88EKNpKQkm4YNGzbq3bt3HYB33nnH3dvbu7G3t3fj9957r1r6fr28vPz69u3r5ePj0/j48ePlFi9e7NyoUSPfBg0aNGrbtq1P+r4PHz7s2Lp16wY1a9Zs8sEHH+Tpm9G0adOkJk2aJAF4eXmluLq6pkZFRRXIybu0AAiRB/sv7qfv9325kHCBlU+tpEudLmaHJApJSgps2XK7x/7evcZyDw947DHjLD8gAFxdzY2zpGndmgaZlz3+ODFvvEF0fDw2/v54Z14/eDCXR4/mSlQUdn36UC/jum3bOHK3MiMjI8tPmzYtMigo6Hr//v29xo8f7/bOO+9cfPnllz3Pnz9vV7169dSZM2dWGT58+OXg4OBrs2fPrhYREXEIIDw8vMKCBQuq7Ny587DWmhYtWvj6+/vHV61aNe306dMO33zzzUl/f//I8+fP27344oteGzZsiGjYsGHyxYsXbzUHHDt2rPyWLVuOxMbG2vr6+vq99tpr0Q4ODnnu2xYWFlYhJSVFNWrUKCmvr82K3KQsxF1orVl3Yh2Pzn+UplObcuXmFdaGrJXKvxQ6fRqmT4fHHzcG4unUCSZMgMqV4eOPYc8eoxf/zJkwYIBU/iXF/fffnxwUFHQdICQk5MqWLVsq2djYMGDAgCszZsxwvXz5su2uXbsq9e/f/1rm127YsKFS9+7dY52dnS0uLi6WHj16XA0LC3MC8PDwSPb3979u3a5i69at4xs2bJgM4O7ufquJPigoKNbR0VF7eHikurq6ppw9ezbPJ9+nTp2yHz58eN0ZM2ZEFtSlBmkBECIbyWnJLNy/kIlbJ7Lv4j7cK7rzfuf3GdVyFFUrVDU7PFEAEhONe/HTz/IPW8cm9fSE4GDjLL9LF3B2NjfO0iSnM3YnJyw5rffwIDU3Z/yZZTWNL8Dzzz9/pUePHvXLly+ve/XqddXe3j5P+61QoYIlN9tlPNu3tbUlNTX1joDmzJlz34cfflgdYPr06ZEdOnS4kXF9TEyMzaOPPlr/7bffPpeecBQESQCEyCTmZgzTdkxj8rbJRCVE0ditMTN7zyS4SbAM61sKHDt2u8IPCzN66js4GCPuPfusUek3bAgylEPpERUVVW7dunUVAwICrs+fP9+1Xbt2CWBcU3d3d08JDQ31WLVq1V/p29vZ2emkpCTl4OCgO3funPD00097vf/++xe01qxcubLy7NmzT2Quo1OnTtfHjh1bOyIiolz6JYCMrQA5GTJkSOyQIUNis1qXmJioevToUX/QoEFXhg8ffjW/70FWJAEQwup4zHE+3/o5M/fM5EbKDQLrBjKrzyyC6gXJwD4l2PXrsGGDUeGvWgXHjxvLvb3hmWeMCr9TJ6hQIae9iJLMy8srcfLkydVGjhxZwdvbO/HVV1+NTl83aNCgK1OmTLFr3rx5Yvqyp556KtrX17eRn5/fjaVLl54MDg6+0rx5c1+AkJCQ6IcffvjmkSNH7hi6qXr16qmTJk2K7Nu3b32LxUKVKlVStmzZcvReY585c2bl7du3V7p69ardggULqlqXnWzXrt3Ne923TAcsyrwtZ7YQ+kcoSw4vwc7GjuAmwYxtO5am7k3NDk3kg9ZGU356hb9pEyQnGxV8ly5Ghd+tG9Srd/d9lXQyHbDRW79nz57eR48ePZjV+iFDhng2a9bsxpgxY0r8lMXZyW46YGkBEGVSmiWNJRFLCP0jlK1nt1K5fGXeeOQNXmz9ItWdqpsdnsija9eMWfPS78s/c8ZY3rgxvPSSMfreI48YTf1CpGvcuLGvo6OjZdq0aWfMjsUMkgDco8fen8Er3XvQqYVUGiVBfFI8s/bM4vOtn3My9iR1K9dl8qOTGf7gcCqWq2h2eCKXLBbjtrz0Cn/LFkhNNTrrBQTAW28ZA/LUqmV2pMJsDRo0SM7u7P/gwYNlekp6SQDuwaxfd/OLZSRp6z6kU4t/mR2OyMG5uHNM+nMS03ZO41rSNdrVaseEoAn0adAHW5uCH71LFLwrV2Dt2tuV/sWLxvJmzeD1141m/TZtII8duYUosyQBuAdRcZepcK0ZU1983uxQRDb2XNhD6B+hfHfgOyzawuO+jzOu7Tja1GxjdmjiLtLSYMeO2z32t20zru+7ut453O79BTZquxBliyQA9+DfAwP598BAs8MQmVi0hVXHVhH6RyjrT66non1FXmj5Aq+0eYU6lWWi9eLswgVYs8ao8NesgZgY43a8hx6Ct982Kv2WLaEQhlwXosyRBCCfJv64iaEBraji4mh2KMIqMTWRefvmMfGPiRy+fJgaTjX4JOATRrYYyX3l7zM7PJGFlBTYuvV2j/3du43l7u7Qq5dR4QcGGqPyCSEKlgwFnA+7/rrIuD1BdB0v1/2Lg8s3LvPexveo/Xltnl32LA52DsztO5cTL5/g9Ydfl8q/mDlzBmbMgH79jNnzOnSATz8FJyf48EMjCTh//vbEO1L5i3tx5MiRct7e3o3NjiM7H374oZunp6efUqpFxkl+LBYLw4YNq+Xp6enn4+PTaPPmzQU+UoW0AOTDqFlfgEMy4x9/wexQyrQjl4/w2dbP+HbvtySmJtLduzvj2o6js1dnGbinGElKuj3c7qpVcNDaH7tWLRg40LhFr0sXcHExN04hzNCxY8eEfv36XevSpcsdkyQtWrTI5cSJE+UjIyMPhIWFVXzhhRc89+3bF1GQZUsLQB5FXohjO1/ieb0fnZv63P0FokBprdkYuZHeC3vTcEpDZu+ZzeAmgzn4wkFWBK+gS50uUvkXAydOwJQpRjO+q6vRjD95sjGT3oQJRhJw6pQx8U7fvlL5i6Jx6NChcr6+vo02btxYITU1leeee66mn5+fr4+PT6Px48dXBePM+7nnnqvp7e3d2MfHp9GMGTMqAyxfvtypZcuWDTp16lTfy8vLLzg42DMtLY3U1FT69evnlb79u+++m6fpfh9++OGbDRo0SM68/JdffrnvqaeeumJjY4O/v//1uLg4u1OnThXoPS7SApBHz02fBuWv8WmXf5odSpmSkpbC4kOLCf0jlJ1RO6laoSpvdXiLF1q9gHsld7PDK/Nu3DCG200/yz9qHQC1bl0YPtw4y+/UCSrKUAtl2tO/PF3rwKUDBdqU7VfN78bMPjPvOpDP3r17HQYNGlRv5syZJ9u2bXtzwoQJVV1cXNIOHDhw+ObNm6pVq1YNe/XqFbd169YK+/fvdzx8+PDBqKgou9atW/sGBQUlAOzfv7/i7t27D/j4+CR36NDBe86cOZXr16+fFBUVZZ8+1sDly5cLpItqVFSUvZeX163EwMPDI/nUqVP2tWvXTimI/YMkAHm2L2YLVWz8Gdje1NE1y4xrideYsWsGk/6cxJm4MzSo0oCpPaYy5IEhONpLB0yzaA0REbcr/I0bjaZ+R0fo3BlGjzY68NWvb3akQkBMTIzdY489Vn/x4sXHW7RokQiwbt0654iIiApLly6tDBAfH2976NCh8uHh4U4DBgyIsbOzo1atWqkPPfRQwubNmyu4uLhYmjRpcr1Ro0bJAAMGDIgJDw+v1LNnz7gzZ844DB06tFavXr2u9e3bN87MY80LSQDy6NzEn4i6WmI+3xLrVOwpvvjzC77e9TXxyfF08urElz2+pLt3d2yUXLkyQ1wcrF9/u9I/dcpY7usL//iHUeG3bw/ly5sbpyi+cnOmXhicnJzSqlevnhwWFlYpPQHQWqvQ0NDT/fr1u+MHfcWKFdlekMpqWmE3N7e0AwcOHFqyZInz1KlT3b7//nvXRYsWRaZvk5qaip+fXyOAbt26xX7++efncxOzh4dHSmRk5K0Jh6KiosoV5Nk/SB+AXEtKTiPi9GVsbBQ1qsgFy8Ky/dx2Bi0eRL1J9Zj05yR6NejFjmd3EDY0jJ4+PaXyL0JaG8PtfvKJ0XxfpYpxvX7BAmjeHKZNg8hIOHQIQkON6/xS+YviyN7eXv/666/HFy5cWGXq1KmuAIGBgde++uort6SkJAWwb98+h7i4OJsOHTrEL1682DU1NZXz58/bbdu2rVL79u2vg3EJICIiolxaWhqLFy92bd++fXxUVJRdWloaw4YNi/3oo4/O7d+//45LHHZ2dkRERByKiIg4lNvKH6B3796x8+fPr2KxWPjtt98qOjk5pRV0AiAtALn0+qyfmXQmhEXdtvDEIw+aHU6pYtEWlh1ZRugfoYSfDsfZwZkxbcYw+qHR1HKRwdyLUkwMrFtn3Je/ejVERRnLH3wQXn3VOMtv2xbKlct5P0IUN87OzpbVq1cf69Spk4+Tk1PamDFjLkdGRjo0adLEV2utXF1dU1auXHk8JCQkdsuWLZV8fX0bK6X0u+++e9bT0zN13759+Pn5XR81apRnZGRk+Xbt2sWFhITEbtu2zXHEiBFeFotFAbz33ntn8xLXBx98UG3y5Mn3X7lyxf6BBx5o1Llz52vff//9qQEDBlxbsWKFS+3atf0cHR0tX3/9dWRBvydFPh2wUqoWMAdwBzQwXWv9RU6vMXs6YItFU2nMQ6TZXyXh4wjs7WQYsoJwI+UG3+75ls+2fsbRmKN4unjyykOvMKL5CJwdnM0Or0ywWGDnztsD8fz5p7GscmVjmN1u3Yxhdz08zI5U5IdMB1xwli9f7hQaGuoeFhZ2zOxY8qo4TQecCozTWu9SSjkBO5VSa7XWh0yIJVc+XbSBm67bGVplqlT+BeBCwgWmbJvCVzu+4srNK7Sq3orv+n1Hv0b9sLORRqnCdumScXa/apUx3O7ly8Zwu61awZtvGpV+69Yy3K4QpV2R/9pqraOAKOvf8Uqpw0ANoPgmAL9/jI2jO5NeG2p2KCXawUsHmfjHRObtn0dKWgq9G/RmXNtxPOL5iNy7X4hSU43hdtM77+3caSyvVs24PS99Up2qVc2NU4jirGfPnvE9e/aMNzuOgmTq6ZZSygtoBvyZxbqRwEgAT0/PIo0ro992neSq61p6lP8Q5wrSwymvtNb8dvI3Qv8IZdWxVTjaOTKi2QjGtBmDdxVvs8Mrtc6evX2Wv3YtXLtmnNG3bQsffGBU/A8+CDbSp1KIMsu0BEApVQn4EXhFa/23++q01tOB6WD0ASji8G7xb16HpckHaO5dw6wQSqTktGQW7l/IxK0T2XdxH+4V3fmg8weMajmKKhVkcPeClpQEv/9++yx//35jeY0a8MQTRoXv7w/3ybQIQggrUxIApZQ9RuU/X2v9kxkx5EZamsbWVtGrTSOzQykxYm7GMG3HNCZvm0xUQhSN3Rozs/dMgpsE42DnYHZ4pcrJk7cr/N9+g+vXwd7euBd//Hijab9xY+P6vhBCZFbkCYAyLvZ+AxzWWk8s6vLzoukbL3HTco1j4+dgYyO/ojk5HnOcz7d+zsw9M7mRcoPAuoHM6jOLoHpBcn2/gNy8aYy4t2qV0Wv/r7+M5V5eMGSIcZbfuTNUqmRqmEKIEsKMK4APAyFAF6XUHuujuwlx5Gj30YscKv81jvaOUvnnYMuZLfT7oR/ek72ZtnMa/Rv1Z++ovawJWUPX+l2l8r8HWsORI/DFF8bZvKurUclPm2aMsf/FF8b6Eyfgyy+NiXek8hfi3r3xxhv3F9S+xo4dW/2tt966pwlL2rdv7+3k5PRg586d7xhcOyIiolzTpk0benp6+vXo0aNuYmJinn5wizwB0Fpv1lorrXVTrfWD1sfKoo7jbp6fNQlsk5n05Ktmh1LspFnSWHxoMW2/acvDMx8m7GQYbzzyBpGvRDL7sdk0dW9qdoglVnw8LF0Kzz9vVPING8Irrxgj7o0aZZz9x8QYLQCjR4OPjzTxC1HQJk2alKeRLywWC2lpaYUVDq+++uqFadOmncy8fOzYsTVffPHFi6dPnz7g4uKS+sUXX+TpXh7pA5yFUxfi+FNPoVZCP/wfkCl/08UnxTPpz0l4T/am/6L+XLp+icmPTubMmDN86P8h1Z2qmx1iiaO10WHv00+hSxdjuN0+fWDePHjgAfjqK+MMPyICPvvMGJTHUeZAEiLXjhw5Uq5OnTqNe/fuXadu3bqNu3XrVjc+Pt5m6dKlTgEBAfXSt1uyZIlzYGBgvRdeeKFGUlKSTcOGDRv17t27DsA777zj7u3t3djb27vxe++9Vy19v15eXn59+/b18vHxaXz8+PFyixcvdm7UqJFvgwYNGrVt2/ZW5XH48GHH1q1bN6hZs2aTDz74IE/TBQP06dMn3tnZ2ZJxmcVi4Y8//nAaPnz4VYCnn376yrJly/LUzVdGXcnCqBnTofw1Pu4qU/4CnIs7x6Q/JzFt5zSuJV2jXa12TAiaQJ8GfbC1kdFi8io21rg1L70D33nr6OBNm8KYMUZz/8MPy3C7onRqPaN1g8zLHvd9POaNR96Ijk+Kt/Gf4/+3+4MHNx18efRDo69ExUfZ9fmuT72M67Y9u+3I3cqMjIwsP23atMigoKDr/fv39xo/frzbO++8c/Hll1/2PH/+vF316tVTZ86cWWX48OGXg4ODr82ePbtaRETEIYDw8PAKCxYsqLJz587DWmtatGjh6+/vH1+1atW006dPO3zzzTcn/f39I8+fP2/34osvem3YsCGiYcOGyRcvXrz143js2LHyW7ZsORIbG2vr6+vr99prr0U7ODjc091tFy9etHNyckqzt7cHwMvLK/nixYt5+tWQBCALnw4eTLXlzgR3KttT/u65sIfQP0L57sB3WLSFx30fZ1zbcbSp2cbs0EoUiwV27bpd4W/dCmlpxi15gYG3h9utIXeaClEo7r///uSgoKDrACEhIVcmTZpUzcbG5uKAAQOuzJgxw/Uf//jHlV27dlX66aef/tbMvmHDhkrdu3ePTT8D79Gjx9WwsDCn/v37x3p4eCT7+/tft25XsXXr1vENGzZMBnB3d791TSAoKCjW0dFROzo6prq6uqacPXvWrl69egU6sU9+SAKQhSZ17ufbl0aaHYYpLNrCqmOrCP0jlPUn11PRviL/aPUPXn7oZepUrmN2eCVGdLQxzO6qVcaAPNHRxvKWLeFf/zIq/YceAjv5HyjKmJzO2J0cnCw5rfdw8kjNzRl/ZllN4wvw/PPPX+nRo0f98uXL6169el1NP5vOrQoVKljuvhVkPNu3tbUlNTX1joDmzJlz34cfflgdYPr06ZEdOnS4cbd9uru7p8bHx9umpKRgb29PZGRkOXd39+S8xC99ADJISk6jztgh/G9ZuNmhFLnE1ES+3vU1fl/60WNBD45cPsInAZ9wduxZPu/2uVT+d5GaClu2wFtvGePou7vD4MFGAhAUBHPnwsWLsH07vP++0cQvlb8QRSMqKqrcunXrKgLMnz/ftV27dgkAXl5eKe7u7imhoaEeI0eOvDVhkZ2dnU6fJrhz584JK1euvC8+Pt4mLi7OZuXKlZU7d+78tyGBO3XqdH3btm1OERER5QAyXgK4myFDhsSmTxmcm8ofwMbGhjZt2sTPmjWrMsDMmTOr9OzZMza3ZYK0ANzh33N+IdJlLpGXe5odSpGJvh7NVzu+Ysr2KVy6fokH73+QuX3nMqDxAMrZykXonJw/f+dwu1evGkPrtmkD775r3LLXvLkMtyuE2by8vBInT55cbeTIkRW8vb0TX3311ej0dYMGDboyZcoUu+bNmyemL3vqqaeifX19G/n5+d1YunTpyeDg4CvNmzf3BQgJCYl++OGHbx45cuSOH8jq1aunTpo0KbJv3771LRYLVapUSdmyZcvRgoi/RYsWDU6cOFH+5s2btu7u7k2//PLLyH79+sWFhoaeHThwYL0PPvigRuPGjW+8/PLLeZp1scinA86PopgO2GLROI19iBS7GK5/fKTUz/p35PIRPtv6Gd/u/ZbE1ES6e3dnXNtxdPbqLPfuZyE2Fo4fNx67dhm34e3bZ6zz8DCa9B99FAICjKl0hSgOZDpgo7d+z549vY8ePXowq/VDhgzxbNas2Y0xY8aU+CmLs1OcpgMulkJ/2sCNytsJua/0TvmrtWbTqU2E/hHKsr+W4WDrQEjTEMa0HUMjt7I93LHFYpzRHz9u3HaXXtmnP2Jibm9rbw+PPAKffGJU/E2ayL34QpREjRs39nV0dLRMmzbtjNmxmEESAKuPwz/Gprw7k18tfVP+pqSlsPjQYkL/CGVn1E6qVqjKWx3e4oVWL+Be6Z4GqCpRkpKMAXUyV+7Hjxvj6icm3t7W1hY8PaFePejf3/g3/VG/PlSsaNphCCHyoEGDBsnZnf0fPHjwcFHHU5xIAoDR/N+pZnecyj+BS8XSM+XvtcRrzNg1g0l/TuJM3BkaVGnA1B5TGfLAEBztS+doMteuZV3BHz8OZ84YA++kq1DBqNAbNDCa7zNW8rVrG2f6Qoh7ZrFYLMrGxqb4X28uhSwWiwKyvFtBEgDAxkbx42svmx1GgTkVzGg0aQAADaxJREFUe4ov/vyCr3d9TXxyPJ28OvFljy/p7t0dG1Wye6RpDVFR2VfyV67cub2bm1Ght29/ZwVfr57RU1+a7oUodAeio6Mbubm5XZMkoGhZLBYVHR3tAhzIan2ZTwDmrdvN2gO7mPzsYJwrluzparef207oH6EsPrQYgIF+AxnbZiwtqrcwObK8SU6GU6eyruBPnDBmxUtnY3O7qb5fvzsr+Lp1wdnZvOMQQkBqauozFy5c+PrChQt+yK3nRc0CHEhNTX0mq5VlOgH4fsM+hqwLxMbiyOuXe9K4Ysm7Hm7RFpYdWUboH6GEnw7H2cGZMW3GMPqh0dRyqWV2eNmKj8/+LP70aaNTXjpHR6Myr1fPuKc+c1O9DJkrRPHVokWLS0Bvs+MQf1dmE4DvwvYSvNofG4sj6waH0bh2yan8LyRcYOf5new4v4P5++dzNOYoni6eTAyayIjmI3B2MP+0V2tj4JvsKvno6Du3r1LFqNDbtjUG0MlYyXt4SFO9EEIUtDKZAGSs/H8bvIGOTevd/UUmuZhwkZ1RRmW/M2onO8/v5Fz8OQAUiodqPsT7nd+nX6N+2NkU7ceZkmKcrWfXVH/9+u1tlYJatYwKvU+fv1+Pd3Ep0tCFEKLMK5MJQNjh/dikVeS3kPXFqvJPr+x3nt95q9LPWNk3qNqATl6daFm9JS08WvDg/Q/i5OBUqDElJGR9X/zx48Z1+oxTYDs43G6q79Llzgrey8tYL4QQongoUyMBxiYkcl8l4za/K3E3qOJc4Z73mV+Xrl+6o6LfGbWTs3FnAaOy96nic6uib1G9Bc3ub1Yolb3WRnN8dk31Fy/euX3lyn8/e09/VK8uw94KUZwUh5EARfFVZloAvtuwh6dW9OSTNnN4tV+XIq38o69H3zqz3xG1g53nd3Im7vbAUw2qNKBD7Q608GhBy+otC7yyT029s6k+8xl9QsKd29esaVToPXr8vZKXYW6FEKJ0KBMJwMINu3lqVQA2VKSVt1ehlnX5xuVbHfR2Rhln+Kevnb613qeKD494PnLr7L6ZR7MC6bR3/XrOTfWpqbe3LVcO6tQxKvQOHe6s4OvUgfKlZywkIYQQ2Sj1CcCtyj+tIr+FbKBj07r3vE+tNTE3Yzh97TSnr53mYPTBW2f4p66durWdt6s37Wq1Y3Tr0bea8V3K3723W3KyMfnM1at3/pvV3+nj11+4cOc+XFyMCr15878PZVujhjHUrRBCiLLLlARAKdUN+AKwBb7WWn9cGOVs2ncyX5V/UmoSZ+PO3qrgbz3ibv99I+XOKZvru9anba22vNj6RR50b4F3xeZYbrrcqrBjdsHiHCryjH9nHOgmK/b2RlN85cpQrZoxIU3mpnpXV7l1TgghRPaKPAFQStkCU4BA4CywXSm1VGt9qKDLate4Nm3LPcf7jw+nifd9HI85TszNGK4mXiU6IYbo+KtEX48hOuEKZ+LOcDb+NFHX/7+9e43RorrjOP79lduyLLi71RgKrmKltdQoKrFYW+OFoGBT+sImmBrR1pjW2LTVWm/tiza92EtMNW1qqZeWxqqVGqsGY6hiTAygYFHwvhWrGBFEYUGtXPz3xTmrw7IL7Lo8s/vM75NMnpkzZ2bOec7AnOfM2XNeZsPWtbucq4kDGUMbTds/y8RtM2j4XxtD327jY5vbeH/DYWxZ38zijXDfRujo2HnM+V2/A2huTktLS/o8/PAP14vh3a03NPjhbmZmH00ZLQDHAe0R8SKApNuAWUC/VwBO/umVLNnxR06962rQbp7IWxuh4yDY1AabzsifhaVjPFu2N7CF9Kdso0aliWQ6l6amNBztkUfu3UN89Gj3ljczs3KVUQEYBxTnXl4DfK5rJEkXABcAtLW19elCn2k9irXtZ9OoVhrVQtOQVkYPbWH0sBbGDGuluaGF5hEtNDU00Ni464O96zJypN+dm5lZfRiwnQAjYi4wF9I4AH05x9xvnwWc1Z/JMjMzqwtlNES/ChRnqRmfw8zMzKxGyqgAPAZMlDRB0nBgNnB3CekwMzOrrJq/AoiI7ZIuAu4n/RngTRHxVK3TYWZmVmWl9AGIiAXAgjKubWZmZuW8AjAzM7OSuQJgZmZWQa4AmJmZVZArAGZmZhWk2N2g9QOEpPXAf/cQbX/gjRokZ6BxvqvF+a6ej5L3gyPigP5MjNWPQVEB2BuSlkXElLLTUWvOd7U439VT5bzbvuVXAGZmZhXkCoCZmVkF1VMFYG7ZCSiJ810tznf1VDnvtg/VTR8AMzMz23v11AJgZmZme8kVADMzswoa9BUASadLek5Su6TLy05PX0g6SNIiSU9LekrSd3J4q6SFkl7Iny05XJKuy3l+UtIxhXPNyfFfkDSnEH6spJX5mOskqfY57Z6kIZL+LenevD1B0tKc1tvztNFIGpG32/P+QwrnuCKHPyfptEL4gLw/JDVLmi/pWUnPSDq+CuUt6Xv5Hl8l6VZJDfVa3pJukrRO0qpC2D4v456uYbaLiBi0C2k64f8AhwLDgSeASWWnqw/5GAsck9dHA88Dk4BfAZfn8MuBX+b1mcB9gICpwNIc3gq8mD9b8npL3vdojqt87Iyy813I/8XA34B78/bfgdl5/XrgW3n9QuD6vD4buD2vT8plPwKYkO+JIQP5/gD+Apyf14cDzfVe3sA4YDUwslDO59ZreQMnAscAqwph+7yMe7qGFy9dl8HeAnAc0B4RL0bEVuA2YFbJaeq1iHgtIh7P65uBZ0j/Wc4iPSjIn1/J67OAeZEsAZoljQVOAxZGxJsR8RawEDg97xsTEUsiIoB5hXOVStJ44Azghrwt4BRgfo7SNd+d38d84NQcfxZwW0S8FxGrgXbSvTEg7w9J+5EeDjcCRMTWiNhIBcqbNAX5SElDgUbgNeq0vCPiYeDNLsG1KOOermG2k8FeARgHvFLYXpPDBq3czHk0sBQ4MCJey7vWAgfm9Z7yvbvwNd2EDwS/BX4AvJ+3Pw5sjIjtebuY1g/yl/dvyvF7+32UbQKwHrg5v/q4QdIo6ry8I+JV4DfAy6QH/yZgOfVf3kW1KOOermG2k8FeAagrkpqAfwDfjYiO4r5cy6+rv9mU9CVgXUQsLzstNTaU1DT8h4g4Gnib1FT7gTot7xbSr9MJwCeAUcDppSaqRLUo43q8j6z/DPYKwKvAQYXt8Tls0JE0jPTwvyUi7szBr+emPvLnuhzeU753Fz6+m/CynQB8WdJLpObaU4BrSc2fQ3OcYlo/yF/evx+wgd5/H2VbA6yJiKV5ez6pQlDv5T0NWB0R6yNiG3An6R6o9/IuqkUZ93QNs50M9grAY8DE3It4OKmj0N0lp6nX8nvNG4FnIuKawq67gc5ev3OAfxbCz8k9h6cCm3KT3/3AdEkt+dfWdOD+vK9D0tR8rXMK5ypNRFwREeMj4hBS2T0YEV8DFgFn5mhd8935fZyZ40cOn517jU8AJpI6SA3I+yMi1gKvSPp0DjoVeJo6L29S0/9USY05XZ35ruvy7qIWZdzTNcx2VnYvxI+6kHrPPk/q/XtV2enpYx6+QGqmexJYkZeZpPedDwAvAP8CWnN8Ab/PeV4JTCmc6+ukTlHtwHmF8CnAqnzM78ijQA6UBTiJD/8K4FDSf+jtwB3AiBzekLfb8/5DC8dflfP2HIUe7wP1/gAmA8tymd9F6uFd9+UN/Bh4Nqftr6Se/HVZ3sCtpL4O20itPt+oRRn3dA0vXrouHgrYzMysggb7KwAzMzPrA1cAzMzMKsgVADMzswpyBcDMzKyCXAEwMzOrIFcArK5J2iFphaQnJD0u6fN7iN8s6cK9OO9Dkqb0MU0LJDX35Vgzs/7iCoDVu3cjYnJEHAVcAfxiD/GbSbPQ7TMRMTPS5D9mZqVxBcCqZAzwFqR5FyQ9kFsFVkrqnDXuauCTudXg1znuZTnOE5KuLpzvq5IelfS8pC92vZiksZIezuda1RlH0kuS9pf0zbxvhaTVkhbl/dMlLc5puyPPEWFm1q88EJDVNUk7SCOrNQBjgVMiYnnndLQR0SFpf2AJaUjZg0kjEh6Rj58B/AiYFhHvSGqNiDclPQQsj4hLJM0ELo6IaV2ufQnQEBE/kzQkX29znvtgSkS8keMNAx4kzeO+mDRG/oyIeFvSZaSR8X6yL78nM6ueoXuOYjaovRsRkwEkHQ/Mk3QEaejVn0s6kTQV8Ti6nzZ1GnBzRLwDEBHF+d07J21aDhzSzbGPATflB/xdEbGihzReSxrn/p48Q+Ik4JE0xDvDSZUCM7N+5QqAVUZELM6/9g8gjRl/AHBsRGzLv8obennK9/LnDrr5txQRD+cKxhnAnyVdExHzinEknUtqdbioMwhYGBFn9TItZma94j4AVhmSDgeGkKaU3Q9Ylx/+J5MewgCbgdGFwxYC50lqzOdo7cX1DgZej4g/ATeQpvwt7j8W+D5wdkS8n4OXACdIOizHGSXpU73LqZnZnrkFwOrdSEmdTe8C5kTEDkm3APdIWkmale9ZgIjYIOkRSauA+yLiUkmTgWWStgILgCv38tonAZdK2gZsIU3ZWnQR0Aosys39yyLi/NwqcKukETneD0kz3JmZ9Rt3AjQzM6sgvwIwMzOrIFcAzMzMKsgVADMzswpyBcDMzKyCXAEwMzOrIFcAzMzMKsgVADMzswr6P/XQgGP1fIkeAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgAAAAEWCAYAAAAQHy/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nOzdeXhN1/rA8e/KIILEGCSIGBJCqEqqaClNpCrGKlXE2NK6bV3qdri3t9W5v6vUUDW1qhRVWq2aqgiiqBqKIGoKIkEMkUHms35/7JOKNEgiyc7wfp7nPHL23mevd59znPWutdfeS2mtEUIIIUTZYmN2AEIIIYQoepIACCGEEGWQJABCCCFEGSQJgBBCCFEGSQIghBBClEGSAAghhBBlkCQAJYRSSiulGt9h/Wyl1H+zPH9eKXVRKZWglKqej/I8rGXa5TfmgmI9hoZFUI67tSzbIiirllJqm1IqXik1ubDLE7mnlPq3Uupzs+MQorBJApCNUipCKZWqlKqRbfl+a4XoYX2+wPq8V7btPrEuH2Z9PkwplWGtWBKUUqeVUl8qpbwKMm6t9XNa63etZdoDU4BArXUlrfWVuyUQ98J6jNsLaF9blFLPZF1mPYZTBbH/bGVFKKUCspRz1lpWRkGXlYNRwGXAWWv98r3uTCnlqpRapZSKyvo9zbLeQSk1XykVp5S6oJQan229v1IqXCl1QykVopSqf68xmUEpNVEp9XUetu+klIrMukxr/YHW+pnbvUaI0kISgJydBp7OfKKUagFUyGG7P4EhWbazA/oDJ7Ntt1NrXQmoDAQAScBepZRPQQSbQ4u1FlAeOFwQ+xeFoj5wROfjTly36ZWxAOuBvrd52UTA01puZ+AVpVRX6/5qAN8D/wWqAXuAZXmNK6+KQ++SEGWa1loeWR5ABPAG8HuWZR8D/wE04GFdtsC6/CJQ1bqsO7AO2A4Msy4bBmzPoZzVwIo7xPEvIBqIAkZYy26cpexZwFogESOpWAC8B3hZl2kgAdgMbLM+T7QueyqH8mytx3MZOAX8w/oaO+v6ysAX1pjOW8uyBbyBZCDDuu9Y6/YO1v2dtb5HswHHLOX1Av4A4jASpq7A+9b9JFv39al126zHXhlYCMQAZ6yflU3W99pa7jWMRO7x27y/izAqzSRrWa8AHtmOeYv1OHdYt/kJqA4stsb9e+b3wbp9U+AX4CpwDOh/m7IXAGlAqnW/Adb3a6r1846y/u1g3b4TEAm8ClwAFt3he2NHlu9pluVRGD1Cmc/fBb6x/j0K2JFlXUXr+9L0Dv9HXgeOWN/nL4HyWdZ3t362sdb3rmW2174KHARSrPFGYHzfD2J8R7/ASGLXAfHARm7+H+sEROYQTwDGdyjV+t4mAAes64cDR637OgWMznacFuv2CYAbRrL0dZb998RIpmOt3wnvbGVPsMZ+HSNxKm9dVwPj/3ms9TsRivW7Kg95FIeH6QEUt0eWH5NjGJWbrfXHtz5/TwDeA+YCz1uXfYvRc5CbBGAEcPE2MXTFqDR9rD9SS/h7AnAdeAijF6d8ZjzW9R5kqcisy/56/W3KfA4IB+phtAJDuLUyXAnMscZTE9id5Yf0b8cIfAKssu7LCaPy/NC6ro01/i7W+OtgrWysP7DPZNtX1mNfCPxo3acHRi/MyCxxpAHPWj+35zEqPnWnzzrL81veN2ssJ4BGGInHEWt5ARgV10LgS+u2FYFzGJWNHXA/RjLV7DZl//V5WZ+/A+yyvrcuGBXnu9Z1nYB04P8wEgXHnPZp3fZvCQBQ1bqsVpZlTwKHrH9PA2Zl208Y0PcO71tYlu/Kr9z87t0PXAIetH4GQ63bO2R57R/W1zpmWbYLo9KvY339Puu+ymMksW9leS9yTACsf08kS+VtXRZk/QwV8AhwA2h9h/39tQ9uJtRdAHuMRPEEUC5L2bsxEodqGInGc9Z1H2IkvvbWRwdu812UhzzMeMgpgNtbhNG93wXjP/X522y3EBiilKqC8ePyQy73H4Xxg5GT/hgVS5jWOhHjBym7H7XWv2qtLVrr5FyWeSf9gala63Na66sYP16AMWAN6Ab8U2udqLW+hFHBD8hpR0ophdGqHKe1vqq1jgc+yLL9SGC+1voXa/zntdbhdwvQeqpjAPC61jpeax0BTAaCs2x2Rms9Txvn8b8CXDEqlvz6Umt9Umt9HaNFelJrvVFrnQ4sx6ikwGj1Rmitv9Rap2ut9wPfAf1yWc4g4B2t9SWtdQzwdrbjsmBUgila66Q8HkMl67/Xsyy7jpFEZa6/zq2yrs/Jp1m+K+9z85TZKGCO1vo3rXWG1vorjJZ+2yyvnW59bdbjmKG1vqi1Po/RUv5Na73f+t1eyc33Oc+01musn6HWWm8FNmBUxrnxFLDG+l1Nw+hdcgTaZzueKOt78RPQyro8DeP7V19rnaa1DtVay+QrotiQc3C3twij67wBRiWfI631dqWUC8YpgtVa6ySj/rurOhjdgjlxA/ZmeX4mh23O5aaQPHDLts+sZdbHaMFEZzk2mzvE4IIxZmJvlu0VRosQjNbf2nzEWMMaR9bYzmC8l5kuZP6htb5hLb8S+Xcxy99JOTzP3Hd94EGlVGyW9XYY36PccOPvx+WW5XnMPSR6CdZ/nTFOr2T+HZ9lvXO212Rdn5Ps35XMWOsDQ5VSL2ZZX45bjyWn701u3+c8U0o9DryF0Zq3wfhuHsrly2/5XLTWFqXUOW7zncPoXcg81kkYyfsG6/dwrtb6o3wcghCFQnoAbkNrfQbjHHI3jAFSd/I18DJ3SBRy0AejpZOTaIxKMpN7TiHmoazcuFOZ5zBacTW01lWsD2etdfPbxHIZ40e7eZbtK2tjIGTm/hrdJo47HddljFZV1hHq7ty+d+ZuCvI9PAdszXK8VbRxRcHzuXx9FH8/rqgsz/Mdq9b6Gsbne1+Wxfdxc5Do4azrlFIVMT6fOw0izf5dyYz1HPB+tvehgtZ6adaQ8nckgNEd/9eAXGuvkMvt9q2UcsDoifkY4xRIFYzkU+W0fQ5u+VysvVv1yMV3ztpL9bLWuiHGOILxSin/u71OiKIiCcCdjQQetXbD38l0jFMF2+60kVLKVinVQCk1A+Pc49u32fRbYJhSqplSqgJG6+VeXQTudC39t8BLSqm6SqmqwGuZK7TW0RjdppOVUs5KKRulVCOl1CNZ9l1XKVXOur0FmAd8opSqCaCUqqOUesy6/RfAcOulZzbWdU3vFqe1W/9b4H2llJP1UrXxGAlYftztPcmL1YCXUipYKWVvfTyglPLO5euXAm8opVyso/LfJI/HpZQqjzFGAMDB+jzTQuv+q1rf62cxxiGA0cXuo5Tqa33Nm8DBu5yW+Yf1u1INo/cr86qBecBzSqkHlaGiUipIKXWn0wl58SdQ3rpPe4xBoA5Z1l8EPJRSmb9t5azrY4B0a29AYLbtqyulKt+mvG+BIOt31R4j0U/BGKNxR0qp7kqpxtak4TrGAFdLbg9UiMImCcAdWM8b7snFdle11pvucH6vnVIqAWPk+BaM7tUHtNY5dkNqrddhjALfjDHgaHN+4s9mIvCVUipWKdU/h/XzgJ+BAxgDsLL3egzB+DHNHPm9AuP8Jtb4DgMXlFKXrctetca+SykVhzGSu4n1+HZjDJb7BOOHcSs3W1nTgCeVUteUUtNziPNFjFbgKYzBlkuA+bl7C/7mQ4xKMVYpNSGf+wCM1h5GxTIAo9V4gZuD9nLjPYzL7w5idE/vsy7Li8wrGsAY0Jn1HPtbGFdbnMF4vydprddbY4/BuHzwfYzP9kFuM74jiyUYSeEp637fs+5rD0Zy8al1XycwBmcWCOtYjDHA5xit8ESMQbqZllv/vaKU2mf9XF7CqMivAQMxBqdm7i8cI/k6Zf0eZD1Vgdb6GDAYmIHRA9UD6KG1Ts1FuJ4Y3/sEYCfwmdY6JG9HLEThUTImRQiRF0qpCIwrNTaaHYsQIv+kB0AIIYQogyQBEEIIIcogOQUghBBClEHSAyCEEEKUQSXiRkA1atTQHh4eZochhBAlyt69ey9rrV3uvqUoi0pEAuDh4cGePXe9Gk8IIUQWSqmc7iIqBCCnAIQQQogySRIAIYQQogwqtARAKTVfKXVJKRWWw7qXlVLaestTIYQQQhSxwuwBWIAxr/0tlFL1MG6ZerYQyxZCCCHEHRRaAqC13kbO091+ArxCwc9mJ4QQQohcKtIxAEqpXsB5rfWBXGw7Sim1Rym1JyYmpgiiE0IIIcqOIksArNPa/htjqtG70lrP1Vr7aa39XFzkMlYhhBCiIBVlD0AjoAFwwDqbWF1gn1KqdhHGIIQQJUJCagIvrXuJ2ORYs0MRpVSRJQBa60Na65paaw+ttQfGHN6ttdYXiioGIYQoCWKTYwlcFMjM32fy69lfzQ5HlFKFeRngUmAn0EQpFamUGllYZQkhRGkRkxjDo189yp6oPXz75LcEeQWZHZIopQrtVsBa66fvst6jsMoWQoiS6Hzcebos6sLp2NOsenoVXRv/7UpqIQpMiZgLQAghSrvT104TsCiAS4mXWD9oPY94PGJ2SKKUkwRACCFMFn45nICFAdxIu8GmIZtoU6eN2SGJMkASACGEMNEfF/4gcFEgSim2DNtCy1otzQ5JlBEyGZAQQphkV+QuOn/VmfJ25QkdHiqVvyhSkgAIIYQJQk6HELAwgOqO1QkdHopXdS+zQxJljCQAQghRxNb8uYZuS7rhUcWD0OGh1K9S3+yQRBkkCYAQQhSh5YeX03tZb5q7NGfLsC24OrmaHZIooyQBEEKIIvLl/i8Z8N0A2tZty6Yhm6hRoYbZIYkyTBIAIYQoAp/u/pQRq0bg38Cf9YPWU7l8ZbNDEmWcJABCCFHIPgz9kBfXvUjvpr356emfqFiuotkhCSEJgBBCFBatNf/e9G/+vfnfDGwxkG+f/BYHOwezwxICkBsBCSFEobBoC/9c/09m7J7BqNaj+CzoM2xtbM0OS4i/SAIghBAFLMOSwTM/PcOCPxbwcruXmdRlEkops8MS4haSAAghRAFKzUhl8PeDWX5kORMfmcibj7wplb8oliQBEEKIApKUlkS/5f1Yc3wNH3f5mJfbv2x2SELcliQAQghRAOJT4un1TS+2RGxhdtBsRvuNNjskIe5IEgAhhLhH15Ku0W1JN34//zsL+yxkcMvBZockxF1JAiCEEPfgUuIlAhcFcvTyUZb3W04f7z5mhyRErkgCIIQQ+RQZF0mXRV04E3uGn57+icBGgWaHJESuSQIghBD5cOraKfwX+nPlxhV+HvwzHep3MDskIfKk0O4EqJSar5S6pJQKy7JsklIqXCl1UCm1UilVpbDKF0KIwnI05igdvuxAXEocm4dulspflEiFeSvgBUDXbMt+AXy01i2BP4HXC7F8IYQocPuj99NxQUcyLBlsHbYVPzc/s0MSIl8KLQHQWm8DrmZbtkFrnW59uguoW1jlCyFEQdtxbgedv+pMBfsKhA4Pxaemj9khCZFvZk4GNAJYd7uVSqlRSqk9Sqk9MTExRRiWEEL83aZTmwhcFEjNijUJHR6KZ3VPs0MS4p6YkgAopf4DpAOLb7eN1nqu1tpPa+3n4uJSdMEJIUQ2Px37iaAlQTSo2oBtw7fhXtnd7JCEuGdFfhWAUmoY0B3w11rroi5fCCHyYlnYMgavHEyr2q1YP2g91StUNzskIQpEkfYAKKW6Aq8APbXWN4qybCGEyKsv9n3B0989Tbu67dg0ZJNU/qJUKczLAJcCO4EmSqlIpdRI4FPACfhFKfWHUmp2YZUvhBD3YtquaTzz0zMENgpk/eD1ODs4mx2SEAWq0E4BaK2fzmHxF4VVnhBCFAStNR+EfsAbIW/Qp2kflvZdioOdg9lhCVHg5E6AQghhpbXm9U2v83+//h/BLYOZ32s+djbyMylKJ/lmCyEEYNEWXlz7Ip/t+YznfJ9jZtBMbJSZV0oLUbgkARBClHnplnSeWfUMXx34igntJvC/Lv9DKWV2WEIUKkkAhBBlWmpGKgO/G8h3R7/jnU7v8EbHN6TyF2WCJABCiDIrKS2Jvt/2Zd2JdUwJnMK4duPMDkmIIiMJgBCiTIpPiafH0h5sO7ONud3n8qzvs2aHJESRkgRACFHmXE26yuOLH2dv1F4WP7GYp1vkdNWyEKWbJABCiDLlYsJFAr8OJPxyON/1/45eTXuZHZIQppAEQAhRZpy7fo6ARQFExkWy+unVdGnUxeyQhDCNJABCiDLh5NWT+C/051ryNTYM3sBD7g+ZHZIQppIEQAhR6h2+dJgui7qQmpHK5iGb8XXzNTskIUwnt7kSQpRqe6P28siCRwDYOmyrVP5CWEkCIIQotX49+yuPLnyUSuUqsW34NprXbG52SEIUG5IACCFKpY2nNhL4dSC1K9UmdHgojas1NjskIYoVSQCEEKXOj+E/ErQkiMbVGrNt2DbqVa5ndkhCFDuSAAghSpWlh5bS99u+tKrdipChIdSqVMvskIQoliQBEEKUGvP2zmPQ94N42P1hNgZvpJpjNbNDEqLYkgRACFEqfLLzE0atHkXXxl1ZO2gtTg5OZockRLEmCYAQokTTWvPu1ncZv2E8fb378sOAH6hgX8HssIQo9gotAVBKzVdKXVJKhWVZVk0p9YtS6rj136qFVb4QovTTWvPqxld5c8ubDL1vKN88+Q3lbMuZHZYQJUJh9gAsALpmW/YasElr7Qlssj4XQog8s2gLY9aMYdKOSYzxG8P8XvOxs5GbmwqRW4WWAGittwFXsy3uBXxl/fsroHdhlS+EKL3SLekM+2EYs/fO5tWHXuXTbp9io+SMphB5UdTpci2tdbT17wvAba/PUUqNAkYBuLu7F0FoQoiSICU9hYHfD+T7o9/zXuf3+HeHf6OUMjssIUoc01JmrbUG9B3Wz9Va+2mt/VxcXIowMiFEcXUj7Qa9vunF90e/Z+pjU/lPx/9I5S9EPhV1D8BFpZSr1jpaKeUKXCri8oUQJVRcShzdl3Rn+9ntfNHzC0bcP8LskIQo0Yq6B2AVMNT691DgxyIuXwhRAl25cYWAhQHsjNzJ0r5LpfIXogAUWg+AUmop0AmooZSKBN4CPgK+VUqNBM4A/QurfCFE6XAh4QJdFnXh+JXjfN//e3o06WF2SEKUCoWWAGitn77NKv/CKlMIUbqcvX6WgIUBnI8/z5qBa/BvKD8fQhQUuWhWCFEsHb9ynIBFAVxPvs4vwb/Qvl57s0MSolSRBEAIUeyEXQojYGEAGTqDkKEh3O96v9khCVHqyJ0zhBDFyp6oPTyy4BFsbWzZOmyrVP5CFBJJAIQQxUbomVAe/epRnB2cCR0eSjOXZmaHJESpJQmAEKJY2HByA499/RhuTm6EDg+lYdWGZockRKkmCYAQwnQrj66kx9IeeFX3YtvwbdR1rmt2SEKUepIACCFM9fXBr+m3vB+tXVsTMjSEmhVrmh2SEGWCJABCCNPM2TOHISuH0LF+RzYM3kBVx6pmhyREmSEJgBDCFJN3TOa5Nc/RzbMbawauwcnByeyQhChTJAEQQhQprTUTt0xkwi8T6NesH98/9T2O9o5mhyVEmSM3AhJCFBmtNRM2TGDKrikMbzWceT3mYWtja3ZYQpRJkgAIIYpEhiWDMWvGMHffXF5s8yJTu07FRkknpBBmkQRACFHo0jLSGPbjMJYcWsLrD7/O+4++j1LK7LCEKNMkARBCFKqU9BSeWvEUPx77kQ8e/YDXO7xudkhCCCQBEEIUosTURPos68Mvp35hxuMzeKHNC2aHJISwkgRACFEoridfp/vS7uw4t4P5Pecz/P7hZockhMhCEgAhRIG7fOMyXb/uyoGLB/im7zf0a97P7JCEENlIAiCEKFDR8dF0WdSFE1dP8MNTPxDkFWR2SEKIHEgCIIQoMGdiz+C/0J8LCRdYN2gdnRt0NjskIcRtSAIghCgQf175k4CFAcSnxrNxyEba1m1rdkhCiDsw5S4cSqlxSqnDSqkwpdRSpVR5M+IQQhSMgxcP0vHLjiSnJxMyNEQqfyFKgLsmAEqpdkqpmUqpg0qpGKXUWaXUWqXUP5RSlfNaoFKqDvAS4Ke19gFsgQF5D10IURzsPr+bTgs6YWdjx7bh22hVu5XZIQkhcuGOCYBSah3wDPAz0BVwBZoBbwDlgR+VUj3zUa4d4KiUsgMqAFH52IcQwmRbI7biv9CfKuWrEDo8lKY1mpodkhAil+42BiBYa30527IEYJ/1MVkpVSMvBWqtzyulPgbOAknABq31huzbKaVGAaMA3N3d81KEEKIIrD+xnj7L+tCgSgN+Cf6FOs51zA5JCJEHd+wByKz8lVIVlTJm7VBKeSmleiql7LNuk1tKqapAL6AB4AZUVEoNzqHsuVprP621n4uLS16KEEIUsu+OfEfPpT3xruHN1mFbpfIXogTK7SDAbUB56/n7DUAwsCCfZQYAp7XWMVrrNOB7oH0+9yWEKGILDyyk/4r++Ln5sXnoZlwqSoIuREmU2wRAaa1vAE8An2mt+wHN81nmWaCtUqqCMqYD8weO5nNfQogiNOv3WQz9YSidPDqxIXgDVcpXMTskIUQ+5ToBUEq1AwYBa6zLbPNToNb6N2AFxhiCQ9YY5uZnX0KIojPp10mMWTuGHl49WDNwDZXKVTI7JCHEPcjtjYDGAq8DK7XWh5VSDYGQ/BaqtX4LeCu/rxdCFB2tNW+GvMl7oe/xVPOnWNRnEfa29maHJYS4R7lKALTW2zDGAWQ+P4VxLb8QohTTWjP+5/FM/W0qI+8fyZzuc7C1yVfnnxCimLnbfQDmKaVa3GZdRaXUCKXUoMIJTQhhpgxLBqN+GsXU36Yy9sGxzO0xVyp/IUqRu/UAzAT+a00CwoAYjBsAeQLOwHxgcaFGKIQocmkZaQz5YQjfhH3DGx3e4J3O72CM2RVClBZ3TAC01n8A/ZVSlQA/jDsBJgFHtdbHiiA+IUQRS05P5qkVT7Hq2Co+8v+IVx9+1eyQhBCFILdjABKALYUbihDCbAmpCfT+pjebTm9iZreZjHlgjNkhCSEKiUwHLIQAIDY5lqAlQeyK3MWCXgsY2mqo2SEJIQqRJABCCGISY3js68cIuxTGsieX8WSzJ80OSQhRyPKUACilKljvCCiEKCWi4qMIWBjA6djT/DjgRx73fNzskIQQRSBXdwJUSrVXSh0Bwq3P71NKfVaokQkhCl1EbAQdvuzAubhzrBu0Tip/IcqQ3N4K+BPgMeAKgNb6ANCxsIISQhS+8MvhPDz/Ya4lXWNj8EY6eXQyOyQhRBHK9SkArfW5bNcBZxR8OEKIonDgwgG6LOqCUootw7bQslZLs0MSQhSx3PYAnFNKtQe0UspeKTUBmcFPiBJpV+QuOn3VCQc7B7YN2yaVvxBlVG4TgOeAfwB1gPNAK+tzIUQJsiViCwELA6juWJ3Q4aE0qdHE7JCEECbJ7Y2ALmNMBSyEKKHWHl9L32/70rBqQ34J/gU3JzezQxJCmChXCYBSqgHwIuCR9TVa656FE5YQoiAtP7ycQd8PokWtFvw8+GdqVKhhdkhCCJPldhDgD8AXwE+ApfDCEUIUtAV/LGDkqpG0q9uONQPXULl8ZbNDEkIUA7lNAJK11tMLNRIhRIGbuXsmL6x7gYCGAfzw1A9ULFfR7JCEEMVEbhOAaUqpt4ANQErmQq31vkKJSghxzz7a/hGvb3qdnk16suzJZZS3K292SEKIYiS3CUALIBh4lJunALT1uRCiGNFa88bmN/hg+wc87fM0X/X+Cntbe7PDEkIUM7lNAPoBDbXWqYUZjBDi3li0hXHrxzF993Sebf0ss4JmYWtja3ZYQohiKLf3AQgDqhRUoUqpKkqpFUqpcKXUUaVUu4LatxBlVYYlg2dWPcP03dMZ13Ycc7rPkcpfCHFbue0BqAKEK6V+59YxAPm9DHAasF5r/aRSqhxQIZ/7EUIAqRmpBK8M5tvD3/JmxzeZ2Gki2W7dLYQQt8htAvBWQRWolKqMMZHQMADraQU5tSBEPiWnJ9NveT9W/7maSV0mMaH9BLNDEkKUALm9E+DWAiyzARADfKmUug/YC4zVWidm3UgpNQoYBeDu7l6AxQtReiSkJtDrm16EnA5hVtAsnvN7zuyQhBAlxB3HACiltlv/jVdKxWV5xCul4vJZph3QGpiltb4fSARey76R1nqu1tpPa+3n4uKSz6KEKL2uJV2jy6IubI3YysI+C6XyF0Lkyd16ACoCaK2dCrDMSCBSa/2b9fkKckgAhBC3F5MYQ+DXgRy+dJjl/ZbTx7uP2SEJIUqYu10FoAu6QK31BYzphTOnIfMHjhR0OUKUVufjztNxQUeOXT7GT0//JJW/ECJf7tYDUFMpNf52K7XWU/JZ7ovAYusVAKeA4fncjxBlyulrp/Ff6M/lG5dZP3g9Het3NDskIUQJdbcEwBaoBBTo9URa6z8Av4LcpxCl3dGYowQsCiA5PZlNQzbxQJ0HzA5JCFGC3S0BiNZav1MkkQghbmt/9H4Cvw7EVtmyZegWWtRqYXZIQogS7m5jAOROIkKYbOe5nXT+qjOOdo6EDg+Vyl8IUSDulgD4F0kUQogcbT69mS6LuuBS0YXQ4aF4Vvc0OyQhRClxxwRAa321qAIRQtxq9Z+r6ba4Gw2qNmDbsG3Ur1Lf7JCEEKVIbicDEkIUoWVhy+izrA8tarVgy9AtuDq5mh2SEKKUye1cAEKIIjJ//3ye/elZHqr3EKsHrsbZwdnskITIt71799a0s7P7HPBBGp1FzQKEpaenP+Pr63sp+0pJAIQoRqb/Np2x68cS2CiQlU+tpIK9TJQpSjY7O7vPa9eu7e3i4nLNxsamwG8uJ27PYrGomJiYZhcuXPgc+NvsvZKNCVFMfBD6AWPXj6VP0z6sGrBKKn9RWvi4uLjESeVf9GxsbLSLi8t1jN6Xv5EeACFMprXm35v+zUe/fsTgloP5steX2NnIf01RathI5W8e63ufY2NffmWEMJFFW3hp3UvM/H0mo31H81nQZ9go6ZgTQhQ++aURwiTplnRGrhrJzN9n8nK7l5kVNEsqfyEK2LFjx8p5eno2NzuO2+nZs5kKG5UAACAASURBVGcDDw8PH09Pz+b9+vXzSElJKbIb8MmvjRAmSM1I5envnmbBHwt4u9PbTOoyCaXkxptClDWDBg26eurUqbBjx44dTk5OVlOnTq1RVGVLAiBEEUtKS6LPsj6sOLKCyYGTefORN6XyF6IIHDlypJy3t3ezrVu3VkhPT2f06NF1fXx8vL28vJpNmjSpBoDFYmH06NF1PT09m3t5eTWbN29eVYDVq1c7+fn5NenUqVNjDw8Pn4EDB7pnZGSQnp5O3759PTK3f/vtt2vmJaannnrquo2NDTY2Nvj5+SVGRkaWK4xjz4mMARCiCMWnxNPzm55sjdjKnO5zGOU7yuyQhCgyI0ZQLyyMAr28xceHG/Pnc+5u2x04cMBhwIABjebPn3+6Xbt2SR9//HGNypUrZ4SFhR1NSkpSDzzwQNMePXrE7dq1q8KhQ4ccjx49ejg6OtquTZs23oGBgQkAhw4dqrh///4wLy+v1I4dO3ouXLiwauPGjVOio6Ptjx8/fhjg8uXLtvk5jpSUFLVs2bLqU6ZMueuxFBRJAIQoIteSrvH44sfZE7WHr5/4moEtBpodkhBlwtWrV+169+7deMWKFSd9fX2TATZu3OgcHh5eYdWqVVUB4uPjbY8cOVI+NDTUqX///lft7OyoV69e+oMPPpiwffv2CpUrV7a0aNEisVmzZqkA/fv3vxoaGlqpe/fucefOnXMYOnRovR49elzv06dPXH5iHDp0qHvbtm0TunbtmlBwR35nkgAIUQQuJlwk8OtAwi+Hs6L/Cno37W12SEIUudy01AuDk5NThpubW2pISEilzARAa60mT558tm/fvrdU2GvWrKl8u/1kP1WnlMLFxSUjLCzsyMqVK51nz57tsmzZsmrLly+PyNwmPT0dHx+fZgBdu3aNnTp1alT2/b788suuly9ftvv5559P3tuR5o2MARCikEXGRdJxQUeOXznO6qdXS+UvRBGzt7fX69atO7l06dLqs2fPrgbQpUuX67NmzXLJHHV/8OBBh7i4OJuOHTvGr1ixolp6ejpRUVF2u3fvrtShQ4dEME4BhIeHl8vIyGDFihXVOnToEB8dHW2XkZHBsGHDYj/88MPzhw4duuUUh52dHeHh4UfCw8OP5FT5T5kypcbmzZsr//DDD6dsbfN19iDfpAdAiEJ08upJAhYFcDXpKhuCN/Cw+8NmhyREmeTs7Gz5+eefT3Tq1MnLyckpY9y4cZcjIiIcWrRo4a21VtWqVUtbu3btyeDg4NgdO3ZU8vb2bq6U0m+//Xaku7t7+sGDB/Hx8Ul87rnn3CMiIsq3b98+Ljg4OHb37t2OI0eO9LBYLArgnXfeicxLXK+88kp9V1fXFD8/P2+A7t27X/v444+jC+M9yE4SACEKyZGYIwQsDCA1I5XNQzbj6+ZrdkhClDlNmjRJzRygV6NGjYywsLCjmes+/fTT88D57K+ZM2dOJPC3itzJySkjJCTkRNZl7dq1Szpy5MjR7NvmVnp6+t78vvZeySkAIQrBvuh9dPyyIxrN1mFbpfIX+XLjBmRkmB2FKK1MSwCUUrZKqf1KqdVmxSBEYfj17K90/qozFctVJHR4KM1rFtubkIliRms4cgQ++QQeewyqVYOdO82OSgB07949Pnvrv6Qz8xTAWOAoIJOdi1Jj46mN9PqmF3Wc6rBpyCbqVa5ndkiimLt+HTZtgvXrjcc56zh5b28YMwZcXMyNT5RepiQASqm6QBDwPjDejBiEKGirjq2i3/J+NKnehA3BG6hdqbbZIYliyGKBP/64WeHv2GF08zs5QUAAvPGG0fqvX9/sSEVpZ1YPwFTgFcDpdhsopUYBowDc3d2LKCwh8mfpoaUErwzG182XdYPWUc2xmtkhiWIkJgZ++cWo8H/+GS5dMpa3bg2vvmpU+O3agb29uXGKsqXIEwClVHfgktZ6r1Kq0+2201rPBeYC+Pn5yVzSotj6fN/njPppFB3qd2D106txcrhtXivKiPR0+O23m638vXuN8/vVqxuVfdeuEBgItWqZHakoy8wYBPgQ0FMpFQF8AzyqlPrahDiEuGdTd03l2Z+e5bHGj7Fu0Dqp/MuwyEj44gvo1w9q1ICHH4YPPoBy5eDtt2H3brh4ERYvhuBgqfyLs9dee63Azt+NHz/e7c0338z3p71jxw7HVq1aNW3cuPEtkxMVhCLvAdBavw68DmDtAZigtR5c1HEIcS+01rwf+j7/DfkvT3g/wZInluBg52B2WKIIpaRAaOjNVv7hw8byOnXgySeNVr6/P1QtsJ9rUVSmT5/u+tFHH13I7fYWiwWtNYVxJ79KlSpZFi1adLpFixYpERER9g888IB3nz594mrUqHHPF4jKfQCEyCOtNa9tfI3/hvyX4JbBLHtymVT+ZcSJE/Dpp9C9u3GJXpcuMGMG1K4NkybBoUPGKP7PPzeSAKn8zXfs2LFyDRo0aN6zZ88GDRs2bN61a9eG8fHxNqtWrXIKCAholLndypUrnbt06dJozJgxdVJSUmyaNm3arGfPng0AJk6cWMvT07O5p6dn83feeadm5n49PDx8+vTp4+Hl5dX85MmT5VasWOHcrFkz7yZNmjRr166dV+a+jx496timTZsmdevWbfHee+/labrgli1bprRo0SIFwMPDI61atWrp0dHRBdJ4N/VOgFrrLcAWM2MQIi8s2sILa19g1p5ZPO/3PJ92+xQbJXl0aZWQAFu23Gzln7RO1dK4MYwYYbTyO3WCihXNjLJkadOGJtmXPfEEV197jZj4eGz8/fHMvn7wYC6/9BJXoqOx69WLRlnX7d7NsbuVGRERUX7OnDkRgYGBif369fOYNGmSy8SJEy+OHTvWPSoqys7NzS19/vz51YcPH3554MCB1xcsWFAzPDz8CEBoaGiFJUuWVN+7d+9RrTW+vr7e/v7+8TVq1Mg4e/aswxdffHHa398/Iioqyu6FF17w2LJlS3jTpk1TL168+Fd3wIkTJ8rv2LHjWGxsrK23t7fPv/71rxgHB4c8j20LCQmpkJaWppo1a5aS19fmRH65hMiFtIw0lhxawgPzHmDWnlm80v4VZnabKZV/KaO10YqfNMnovq9eHXr0gC+/NK7L//RTOH7ceMyYAUFBUvmXBLVr104NDAxMBAgODr6yY8eOSjY2NvTv3//KvHnzql2+fNl23759lfr163c9+2u3bNlSqVu3brHOzs6WypUrW4KCgq6FhIQ4Abi6uqb6+/snWrer2KZNm/imTZumAtSqVeuvLvrAwMBYR0dH7erqml6tWrW0yMjIPDe+z5w5Yz98+PCG8+bNiyioUw0yF4AQdxCbHMu8vfOYvns6kXGRNKnehAW9FjDkviF/mxpUlEzXrsHGjTdb+VHW+dpatICXXjJa+Q8/DA5ylqdA3KnF7uSE5U7rXV1Jz02LP7ucpvEFeP75568EBQU1Ll++vO7Ro8c1+zxeh1mhQgVLbrbL2tq3tbUlPT39loAWLlxY5YMPPnADmDt3bkTHjh1vZF1/9epVm8cff7zxW2+9dT4z4SgIkgAIkYPT104z7bdpfLH/CxJSE+js0ZnZQbN53PNxafWXcBkZxmV5mRX+b78ZN+epUsU4p595iV7dumZHKgpKdHR0uY0bN1YMCAhIXLx4cbX27dsngHFOvVatWmmTJ092Xb9+/Z+Z29vZ2emUlBTl4OCgO3funDBixAiPd99994LWmrVr11ZdsGDBqexldOrUKXH8+PH1w8PDy2WeAsjaC3AnQ4YMiR0yZEhsTuuSk5NVUFBQ4wEDBlwZPnz4tfy+BzmRBECILHZF7mLKzil8d/Q7bJQNA3wGMK7tOFq7tjY7NHEPLlyADRuMCn/DBrhyBZQCPz/4z3+MSr9NG7CTX8RSycPDI3nGjBk1R40aVcHT0zN5woQJMZnrBgwYcGXmzJl2rVu3Ts5cNmjQoBhvb+9mPj4+N1atWnV64MCBV1q3bu0NEBwcHPPQQw8lHTt2rFzWMtzc3NKnT58e0adPn8YWi4Xq1aun7dix4/i9xj5//vyqv//+e6Vr167ZLVmypIZ12en27dsn3eu+ldbF/x47fn5+es+ePWaHIUqpDEsGP4T/wJRdU9hxbgdVyldhtO9oXmjzAnWdpRlYEqWlGbfY/flno9Lfv99YXrOmUdl37Wq09mvUMDfOwqaU2qu19jMzhgMHDkTcd999l80q/9ixY+W6d+/umTklcHZDhgxxv//++2+MGzfOtBgL24EDB2rcd999HtmXS74ryqyE1ATm75/P1F1TOR17mgZVGjC963SG3z+cSuUqmR2eyKMzZ25262/aBPHxRou+fXvjhjxdu8J994GNnMERVs2bN/d2dHS0zJkz55zZsZhBEoB7kJamaTT6dV7y78+EQdJFXFJExkXy6e5PmbN3DrHJsbSv155JXSbRu2lvbG0K/kYeonAkJcG2bTcr/fBwY7m7Ozz9tFHhP/ooVK5sbpzCXE2aNEm9Xev/8OHDR4s6nuJEEoB78M7CrZyr/3+cTPYAJAEo7vZH72fKril8E/YNFm2hr3dfxrcbT9u6bc0OTeSC1nDs2M0JdbZsgeRkY3R+p04werRR6TdpYpzfF0LcmSQA96LcDaomtOPjwUPNjkTchkVbWHt8LVN2TiEkIoRK5SrxwgMv8NKDL9GgagOzwxN3ERcHmzffbOWfOWMsb9LkZoXfsSNUqGBunEKURJIA3IN3g7vxLt3MDkPkICktiUUHF/HJrk8IvxxOXee6/C/gfzzr+yxVylcxOzxxGxYLHDhwc/Der78aM+tVqgQBAfD668Zseh4eZkcqRMknCUA+/XfBBsb27kCNKo5mhyKyuJhwkc9+/4zP9nzG5RuXae3amsVPLKZfs37Y28pk68XR5cvwyy83u/YvXjSWt2oFEyYYrfx27YxZ9YQQBUfGw+bDj9tO897px3l61vtmhyKsDl86zDOrnqH+1Pq8u+1d2tdrz5ahW9jz7B4GthgolX8xkp4OO3fCW2/Bgw8al+YNHAirVxuD9hYsMO7Gt38/fPghPPKIVP4i/44dO1bO09Ozudlx3M4HH3zg4u7u7qOU8s06yY/FYmHYsGH13N3dfby8vJpt3769wE90SQ9APry8YjJUsWXG0OfNDqVM01qz6fQmJu+czPoT63G0c2R4q+GMazcOr+ped9+BKDLnz9/s1t+40bj9ro2NkQBMnGi08n19oRBmUxWiWHvkkUcS+vbte/3RRx+9ZZKk5cuXVz516lT5iIiIsJCQkIpjxoxxP3jwYHhBli09AHn0W1gMJ53n01IH09StjtnhlEkp6Sl89cdXtJrTii6LurA/ej/vdn6Xs+POMqv7LKn8i4GUFGPw3iuvQMuWxm11R440zun37g3LlkFMjHGznjffNO7CJ5W/KGxHjhwp5+3t3Wzr1q0V0tPTGT16dF0fHx9vLy+vZpMmTaoBRst79OjRdT09PZt7eXk1mzdvXlWA1atXO/n5+TXp1KlTYw8PD5+BAwe6Z2RkkJ6eTt++fT0yt3/77bfzNN3vQw89lNSkSZPU7Mt//PHHKoMGDbpiY2ODv79/YlxcnN2ZM2cKtCtTegDy6B9fzYCKycx46l9mh1LmXE26yuw9s/l096dEJ0TjU9OH+T3nM7DFQBzsZKYWs508ebOVv3kzJCaCvT106AD/+5/RyvfxkUv0yrIRP46oF3YprEC7sn1q+tyY32v+XW/kc+DAAYcBAwY0mj9//ul27dolffzxxzUqV66cERYWdjQpKUk98MADTXv06BG3a9euCocOHXI8evTo4ejoaLs2bdp4BwYGJgAcOnSo4v79+8O8vLxSO3bs6Llw4cKqjRs3TomOjrbPvNfA5cuXCySVjY6Otvfw8PgrMXB1dU09c+aMff369dMKYv8gCUCeWCwQkbqX+ra96Ojd1OxwyozjV44zdddUFhxYwI20GwQ2CmRB7wV0adhFZuQzUWKicS1+5iV6J04Yyxs0gKFDjQq/c2djBL8QZrp69apd7969G69YseKkr69vMsDGjRudw8PDK6xataoqQHx8vO2RI0fKh4aGOvXv3/+qnZ0d9erVS3/wwQcTtm/fXqFy5cqWFi1aJDZr1iwVoH///ldDQ0Mrde/ePe7cuXMOQ4cOrdejR4/rffr0iTPzWPNCEoA8sLGBmKmruZpQYLMxitvQWrP97HYm75zMqmOrsLe1Z1CLQYxvNx6fmj5mh1cmaQ1Hjtys8Ldtg9RUcHQ0Bu+NHWtcote4sbTyRc5y01IvDE5OThlubm6pISEhlTITAK21mjx58tm+ffveUmGvWbPmtveOzGlaYRcXl4ywsLAjK1eudJ49e7bLsmXLqi1fvjwic5v09HR8fHyaAXTt2jV26tSpUbmJ2dXVNS0iIuKv4a/R0dHlCrL1DzIGINdi49I4HBGDUorqTtKkKSzplnS+CfuGNp+3oeOCjmw/u53/dPgPZ/55hvm95kvlX8RiY2HFCnjmGeMWuz4+xqV50dHw4ovG5XtXrxoj+F94ATw9pfIXxY+9vb1et27dyaVLl1afPXt2NYAuXbpcnzVrlktKSooCOHjwoENcXJxNx44d41esWFEtPT2dqKgou927d1fq0KFDIhinAMLDw8tlZGSwYsWKah06dIiPjo62y8jIYNiwYbEffvjh+UOHDt1yisPOzo7w8PAj4eHhR3Jb+QP07NkzdvHixdUtFgubNm2q6OTklFHQCYD0AOTSmFlLWRr/HL/020PAfc3MDqfUuZ58nc/3fc703dM5e/0sXtW9mBU0iyH3DaGCvdzmrahYLLBv381W/q5dkJFh3E8/IMC4dO+xx6BePbMjFSJvnJ2dLT///POJTp06eTk5OWWMGzfuckREhEOLFi28tdaqWrVqaWvXrj0ZHBwcu2PHjkre3t7NlVL67bffjnR3d08/ePAgPj4+ic8995x7RERE+fbt28cFBwfH7t6923HkyJEeFotFAbzzzjuReYnrvffeqzljxozaV65csb/vvvuade7c+fqyZcvO9O/f//qaNWsq169f38fR0dHy+eefRxT0e1Lk0wErpeoBC4FagAbmaq2n3ek1Zk8HnJxiwemVljiUsyH+fwfkvHMBOhN7hmm/TePzfZ8TnxrPI/Uf4eV2LxPkFYSNkg6qonDpEmzYcPNGPJetk6L6+d2cOvfBB42Z9UTJItMBF5zVq1c7TZ48uVZISMgJs2PJq+I0HXA68LLWep9SygnYq5T6RWt9xIRYcuX1+WtJr3aYsQ0WSeVfQHaf383knZP57sh3ADzl8xTj247H183X5MhKv7Q0o2Wf2crft89Y7uJys8Lv0sW4QY8QovQq8gRAax0NRFv/jldKHQXqAMUyAbBYYO7h/8O+gjsfDHzK7HBKtAxLBquOrWLKrilsP7udyg6VGd9uPC+2eZF6laVPuTCdPXvrjXji4ozr7tu3h/feMyr9++83BroKIf6ue/fu8d27d483O46CZGqnnlLKA7gf+C2HdaOAUQDu7u5FGldW328L54bLdgZXn0Y5O7mdbH4kpiby5R9fMnXXVE5eO4lHFQ+mPjaVEfePwMnByezwSqXkZGOUfmalf8SaXterB089ZVT4/v7GuX0hRNlkWgKglKoEfAf8U2v9t+smtdZzgblgjAEo4vD+8mSnpqytdIR23uYlISVVVHwUM36bwZy9c7iWfI22ddvyUcBH9G7aGzsbOaFckLSG48dvdutv2QJJSeDgYEyXO3KkUel7e8sofSGEwZRfYaWUPUblv1hr/b0ZMeRGapqFcvY2PO7nbXYoJcqBCweYsmsKSw8tJUNn0KdpH8a3G0/7eu3NDq1UiY+HkJCblf7p08ZyLy949lmjwn/kEaggF1EIIXJQ5AmAMkbRfQEc1VpPKery88Lr5VHYOaRx/H8LZPDfXVi0hfUn1jNl5xQ2nd5ERfuKPO/3PGPbjqVh1YZmh1cqaA0HD96s8H/91RjQV7Gi0Z3/r38Zl+g1lLdbCJELZgz5eQgIBh5VSv1hfXQzIY472rIvkjNVFlKtQmWp/O8gOT2ZeXvn4fOZD0FLggi/HM7/Bfwf58adY9rj06Tyv0dXrxoT5wwfDnXqQKtW8Nprxmx648cbPQBXr8KPP8Lzz0vlL0RBeO2112oX1L7Gjx/v9uabb9a6l3106NDB08nJqVXnzp0bZ10eHh5ermXLlk3d3d19goKCGiYnJ+epsjLjKoDtQLGvUV9c8glUtPBZ8HizQymWLiVeYtbvs5j5+0xibsTQqnYrFvVZRP/m/SlnK5O351dGBvz++83Be7t3G1eiVK0KgYFGt35gILi5mR2pEKXX9OnTXT/66KMLud3eYrGgtca2kKa0nDBhwoXExESbefPmuWRdPn78+LovvPDCxVGjRl0bOHCg+7Rp02q8+uqrMbndr1z0k4PDp64S5jAHz5QB+DX2MDucYuVozFFG/TQK90/cmbh1Ig/WfZDNQzazb9Q+BrccLJV/PkRHw4IFMGCAce19u3bw9tvGuv/+F3buNKbO/eYbGDZMKn8h8uLYsWPlGjRo0Lxnz54NGjZs2Lxr164N4+PjbVatWuUUEBDQKHO7lStXOnfp0qXRmDFj6qSkpNg0bdq0Wc+ePRsATJw4sZanp2dzT0/P5u+8807NzP16eHj49OnTx8PLy6v5yZMny61YscK5WbNm3k2aNGnWrl27v+YlP3r0qGObNm2a1K1bt8V7772X5zts9OrVK97Z2dmSdZnFYmHnzp1Ow4cPvwYwYsSIKz/99FOVvOxXhmLn4Pn5n0G5RD4JesXsUIoFrTWbT29myq4prD2+lvJ25RnWahj/bPtPmtaQWRHzKjUVduy4eS7/wAFjee3a0LOn0coPCIDq1c2NU4jC0GZemybZlz3h/cTV1x5+LSY+Jd7Gf6G/Z/b1g1sOvvzSgy9diY6Ptuv1Ta9GWdftfnb3sbuVGRERUX7OnDkRgYGBif369fOYNGmSy8SJEy+OHTvWPSoqys7NzS19/vz51YcPH3554MCB1xcsWFAzPDz8CEBoaGiFJUuWVN+7d+9RrTW+vr7e/v7+8TVq1Mg4e/aswxdffHHa398/Iioqyu6FF17w2LJlS3jTpk1TL168+Fd3wIkTJ8rv2LHjWGxsrK23t7fPv/71rxgHB4d7urrt4sWLdk5OThn29sbl6R4eHqkXL17MUwtMEoAcfPbsM3y63o0gv5Zmh2Kq1IxUloUtY8quKfxx4Q9qVqzJ253e5nm/53Gp6HL3HYi/nD59s8LfvBkSEoxb6z78MHz0kVHpt2wpl+gJURhq166dGhgYmAgQHBx8Zfr06TVtbGwu9u/f/8q8efOq/eMf/7iyb9++St9///3p7K/dsmVLpW7dusVmtsCDgoKuhYSEOPXr1y/W1dU11d/fP9G6XcU2bdrEN23aNBWgVq1aGZn7CAwMjHV0dNSOjo7p1apVS4uMjLRr1KhRgU7skx+SAOTAp35tZo8eYXYYprmadJW5e+cyY/cMouKjaObSjM97fM6gloMob1fe7PBKhBs3YOvWm5X+n38ayz08IDjYqPA7dwYnuQ+SKGPu1GJ3cnCy3Gm9q5Nrem5a/NnlNI0vwPPPP38lKCiocfny5XWPHj2uZbamc6tChQqWu28FWVv7tra2pKen3xLQwoULq3zwwQduAHPnzo3o2LHjjbvts1atWunx8fG2aWlp2NvbExERUa5WrVqpeYlfxgBkkZiUTt2xT/PZ6u1mh2KKk1dP8uLaF6n3ST1e3/Q6zV2as27QOsKeD2Nk65FS+d+B1sbd9j75xLgUr1o16NYN5s6FRo1g2jQ4dgxOnYLPPjO6+qXyF6JoREdHl9u4cWNFgMWLF1dr3759AoCHh0darVq10iZPnuw6atSovyYssrOz05nTBHfu3Dlh7dq1VeLj423i4uJs1q5dW7Vz585/uyVwp06dEnfv3u0UHh5eDiDrKYC7GTJkSGzmlMG5qfwBbGxsaNu2bfyXX35ZFWD+/PnVu3fvHpvbMkF6AG4x/vMVnK/2DTGJZeee/1prdpzbweSdk/kh/AfsbOwY2GIg49uNp2Wtsn0K5G6uX4dNm2628s+dM5Y3awZjxhit/A4dwNHR3DiFKOs8PDySZ8yYUXPUqFEVPD09kydMmPDXSPkBAwZcmTlzpl3r1q2TM5cNGjQoxtvbu5mPj8+NVatWnR44cOCV1q1bewMEBwfHPPTQQ0nHjh275Xy7m5tb+vTp0yP69OnT2GKxUL169bQdO3YcL4j4fX19m5w6dap8UlKSba1atVp+9tlnEX379o2bPHly5FNPPdXovffeq9O8efMbY8eOzdOsi0U+HXB+FMV0wBkZmoovtwa7ZBL/dxjbUj4rSrolne+Pfs/knZPZfX43VctX5Xm/53mhzQu4OrmaHV6xk5oKZ87AyZPG7Hnr1xsD+TIywNnZGLTXtavR+jdx6gohbiHTARuj9bt37+55/PjxwzmtHzJkiPv9999/Y9y4cSV+yuLbKU7TARdL7y75hZSqfzC69heluvKPS4nji31fMO23aZy5fobG1Rozs9tMht43lIrlKpodnqni4owKPqfHuXPG9fiZWreGV181Kv22bSGPpw6FEMVA8+bNvR0dHS1z5sw5Z3YsZpAEAOP87Se/f4StoxuThw4yO5xCcfb6Wab/Np15++YRlxJHB/cOTOs6je5e3bG1KZybVxQ3WhvX3Get2E+duvn35Wz5f40axvn7hx4y/s18NG1qrBNCFH9NmjRJvV3r//Dhw0eLOp7iRBIAwGLRBLr3orqTMxXLO5gdToHaE7WHyTsns/zwcgD6Ne/H+LbjeaDOAyZHVjhSUyEi4taKPWtln5R0c1sbG6O7vlEj6NPn1kq+USOja18Icc8sFotF2djYFP/zzaWQxWJRQI5XK0gCANjaKpZPGGt2GAUmEIJRnwAADTJJREFUw5LB6j9XM3nnZELPhuLs4Mw/2/6Tlx58CffKJf8EdV666h0djfvjN2pk3EI3awVfvz6UkxsXClHYwmJiYpq5uLhclySgaFksFhUTE1MZCMtpfZlPAOb89Duhxw8wc1QwlSuV7NZ/YmoiXx34ik92fcKJqyeoX7k+UwKnMLL1SJwdSk5ztiC66jMrfVdXubmOEGZKT09/5sKFC59fuHDBB7n0vKhZgLD09PRnclpZphOAL9bu47lfH8PO4szEuF5UrlQy724XHR/Np7s/Zfbe2VxNukqbOm1Y9uQynvB+Ajub4vkRZx1Vn9uu+oYNpateiJLG19f3EtDT7DjE3xXP2qEIzF+3j2e2BWCX4cyvz26hsVvJqvxjk2PZG7WXRQcXseTQEtIt6fRu2puX271M+3rti8UUxtJVL4QQxVeZTAC+XL+PkVtvVv5tvDzMDumOridfZ1/0PvZG72VP1B72Ru/lxNUTAFSwr8Bo39H8s+0/aVSt0V32VLDutas+s8KXrnohhCh6ZTIB2HkyDPuMKoQ+s7nYVf5xKXHsj97/V0W/J2oPx6/evJlU/cr18XXzZUSrEfi6+fJgnQepXL5yocWT1676evVyHlXfsCFULrwwhRBC5FGZuhPgtfhkqjoZ97OPTUiiSiVz79EanxLP/gv72Ru1lz3Re9gbtZc/r/yJxvhM6jnXw8/ND19XX+NfN19qVCj4C9Dz21Wf/SFd9UIUL8XhToCi+CozPQBfb9zP0J978L92C3n5iUeLvPJPSE3gjwt/3FLZh18O/6uyr+tcF19XXwa3HIyvqy++br7UrFizQMrO3lWf/Rr523XVt2//90peuuqFEKJ0KBMJwOJN+xmyMQAbKvFgE49CL+9G2g3+uPDHLd344ZfDsWijKe3m5Iafmx8DfAb81cKvVanWPZUpXfVCCCHyotQnAEtC9hP8SwA2GZUIGRrCw80bFti+tdbEpcRxJObILQP0jsQc+auyr12pNn5ufvRr1u+vyv5uk+1obVTY167d/REVlftR9ZnPPTykq14IIco6UxIApVRXYBpgC3yutf6oMMrZdvA0g382Kv/NQ0Lo4JP7yt+iLcQkxhAZF8n5+PPGv3HniYy3/mtdnpCa8NdralasiZ+bH080fYLWrr40q+JH+TS3vyrr2KPw847cVeypqXeOz9kZqlaFWrWkq14IIUTeFXkCoJSyBWYCXYBI4Hel1Cqt9ZGCLqt98/q0tR/NB0+P5IGmrlxIuMD15OvEpcRxLek6VxKtj4RrRMZFERkXyYXE81y4EcnllCjSddot+7PBFiflhpOuQ8WMFjRO64pDSl3KJTTG9qLf/7d3rzF6VHUcx78/WtrdhfbZriCpW2iLgqaSWKDBIkKgNFyKsb7ApAZDQQlRQryACAi+0HhBJUSIRgLlYg0CUggCKSEVSkgILW2x0HJfKEoRKJfeBAJL+fvinIXputt2l92d3ZnfJzl5Zs6cmTnnOdPOec6cncPbr7bTsVGs2AibNkFnZy8ZI92cG410E+8K7e3br3cPra3ps9GA0ZXvuzEzs8FUxm3kMKAjIp4HkHQTMBcY8AbArF9czMPvX80xt14Kt+/gbgzQ2QxbJuVwZPrc2p7X0+cHb32SzTGKzaQu9OZmaGqCceM+ukFPmbzjm3hXGD8+PYs3MzMrQxkNgHagOPfyeuCL3RNJOhM4E2C//fo3gc2BEz7Pf547mSaNp3m3Bi27NWgZ3WDPHMbt3mD82AaNsa20tTRoaRFNTenG3hV6Wm9qglH1mEHXzMwqath2JEfEVcBVkN4D0J9jLPjeKcApA5ktMzOzSiijE/olYN/C+qQcZ2ZmZkOkjAbACuAASVMljQHmAXeUkA8zM7PaGvJHABHxvqSzgXtIfwZ4bUQ8PtT5MDMzq7NSxgBExGJgcRnnNjMzs3IeAZiZmVnJ3AAwMzOrITcAzMzMasgNADMzsxpSRL/esTOkJL0G/GsnyfYCXt9JmipyuevF5a6fj1P2yRGx90BmxqpjRDQAdoWklRExo+x8DDWXu15c7vqpc9ltcPkRgJmZWQ25AWBmZlZDVWoAXFV2BkricteLy10/dS67DaLKjAEwMzOzXVelHgAzMzPbRW4AmJmZ1dCIbwBIOkHS05I6JF1Qdn76Q9K+kpZKekLS45K+n+PbJC2R9Gz+nJDjJemKXObHJB1SONb8nP5ZSfML8YdKWpP3uUKShr6kPZM0StI/Jd2V16dKWp7zenOeNhpJY/N6R94+pXCMC3P805KOL8QPy+tDUqukRZKekvSkpMPrUN+Sfpiv8bWSbpTUVNX6lnStpA2S1hbiBr2OezuH2f+JiBEbSNMJPwfsD4wBHgWmlZ2vfpRjInBIXh4HPANMA34LXJDjLwB+k5fnAHcDAmYCy3N8G/B8/pyQlyfkbQ/ntMr7nlh2uQvlPwf4K3BXXv8bMC8vXwl8Ny+fBVyZl+cBN+flabnuxwJT8zUxajhfH8CfgTPy8higter1DbQD64DmQj2fVtX6Bo4CDgHWFuIGvY57O4eDQ/cw0nsADgM6IuL5iHgPuAmYW3Ke+iwiXo6IR/LyVuBJ0n+Wc0k3CvLn1/LyXGBhJMuAVkkTgeOBJRHxZkRsBJYAJ+Rt4yNiWUQEsLBwrFJJmgScBCzI6wJmAYtyku7l7vo+FgHH5vRzgZsi4t2IWAd0kK6NYXl9SGqQbg7XAETEexGxiRrUN2kK8mZJo4EW4GUqWt8R8QDwZrfooajj3s5htp2R3gBoB14srK/PcSNW7uY8GFgO7BMRL+dNrwD75OXeyr2j+PU9xA8Hvwd+DHyQ1z8BbIqI9/N6Ma8fli9v35zT9/X7KNtU4DXguvzoY4GkPah4fUfES8ClwL9JN/7NwCqqX99FQ1HHvZ3DbDsjvQFQKZL2BG4FfhARW4rbciu/Un+zKekrwIaIWFV2XobYaFLX8J8i4mDgLVJX7YcqWt8TSL9OpwKfAvYATig1UyUaijqu4nVkA2ekNwBeAvYtrE/KcSOOpN1JN/8bIuK2HP1q7uojf27I8b2Ve0fxk3qIL9sRwFclvUDqrp0FXE7q/hyd0xTz+mH58vYG8AZ9/z7Kth5YHxHL8/oiUoOg6vU9G1gXEa9FRCdwG+kaqHp9Fw1FHfd2DrPtjPQGwArggDyKeAxpoNAdJeepz/JzzWuAJyPissKmO4CuUb/zgb8X4k/NI4dnAptzl989wHGSJuRfW8cB9+RtWyTNzOc6tXCs0kTEhRExKSKmkOruvog4BVgKnJyTdS931/dxck4fOX5eHjU+FTiANEBqWF4fEfEK8KKkz+aoY4EnqHh9k7r+Z0pqyfnqKnel67uboajj3s5htr2yRyF+3EAaPfsMafTvRWXnp59l+DKpm+4xYHUOc0jPO+8FngX+AbTl9AL+mMu8BphRONa3SIOiOoDTC/EzgLV5nz+Q3wI5XAJwNB/9FcD+pP/QO4BbgLE5vimvd+Tt+xf2vyiX7WkKI96H6/UBTAdW5jq/nTTCu/L1DfwMeCrn7S+kkfyVrG/gRtJYh05Sr8+3h6KOezuHg0P34FcBm5mZ1dBIfwRgZmZm/eAGgJmZWQ25AWBmZlZDbgCYmZnVkBsAZmZmNeQGgFWapG2SVkt6VNIjkr60k/Stks7ahePeL2lGP/O0WFJrf/Y1MxsobgBY1b0TEdMj4gvAhcCvd5K+lTQL3aCJiDmRJv8xMyuNGwBWJ+OBjZDmXZB0b+4VWCOpa9a4S4BP516D3+W05+c0j0q6pHC8r0t6WNIzko7sfjJJEyU9kI+1tiuNpBck7SXpO3nbaknrJC3N24+T9FDO2y15jggzswHlFwFZpUnaRnqzWhMwEZgVEau6pqONiC2S9gKWkV4pO5n0RsKD8v4nAj8FZkfE25LaIuJNSfcDqyLiXElzgHMiYna3c58LNEXELyWNyufbmuc+mBERr+d0uwP3keZxf4j0jvwTI+ItSeeT3oz388H8nsysfkbvPInZiPZOREwHkHQ4sFDSQaRXr/5K0lGkqYjb6Xna1NnAdRHxNkBEFOd375q0aRUwpYd9VwDX5hv87RGxupc8Xk56z/2deYbEacCD6RXvjCE1CszMBpQbAFYbEfFQ/rW/N+md8XsDh0ZEZ/5V3tTHQ76bP7fRw7+liHggNzBOAq6XdFlELCymkXQaqdfh7K4oYElEfKOPeTEz6xOPAbDakPQ5YBRpStkGsCHf/I8h3YQBtgLjCrstAU6X1JKP0daH800GXo2Iq4EFpCl/i9sPBX4EfDMiPsjRy4AjJH0mp9lD0oF9K6mZ2c65B8CqrllSV9e7gPkRsU3SDcCdktaQZuV7CiAi3pD0oKS1wN0RcZ6k6cBKSe8Bi4Gf7OK5jwbOk9QJ/Jc0ZWvR2UAbsDR396+MiDNyr8CNksbmdBeTZrgzMxswHgRoZmZWQ34EYGZmVkNuAJiZmdWQGwBmZmY15AaAmZlZDbkBYGZmVkNuAJiZmdWQGwBmZmY19D83l2qfJjqqcgAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] @@ -425,35 +396,25 @@ } ], "source": [ - "plot_absolute_time(results, [2, 10], max_batch_size=max_batch_size)" + "plot_absolute_time(experiments_eq, results, [2, 10], max_batch_size=max_batch_size)" ] }, { "cell_type": "markdown", - "id": "6e920708", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ - "#### $N_\\text{ref} \\gg N_\\text{test}$\n", + "#### 2. $N_\\text{ref} >> N_\\text{test}$\n", "\n", - "Now we check whether the speed improvements still hold when $N_\\text{ref} \\gg N_\\text{test}$ ($N_\\text{ref} / N_\\text{test} = 10$) and a large part of the kernel can already be computed at initialisation time of the PyTorch (but not the KeOps) detector." + "Now we check whether the speed improvements still hold when $N_\\text{ref} >> N_\\text{test}$ ($N_\\text{ref} / N_\\text{test} = 10$) and a large part of the kernel can already be computed at initialisation time of the PyTorch (but not the KeOps) detector." ] }, { "cell_type": "code", "execution_count": 8, - "id": "a75794e8", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "metadata": {}, "outputs": [], "source": [ - "experiments = {\n", + "experiments_neq = {\n", " 'keops': {\n", " 0: {'n_ref': 2000, 'n_test': 200, 'n_runs': 10, 'n_features': 2},\n", " 1: {'n_ref': 5000, 'n_test': 500, 'n_runs': 10, 'n_features': 2},\n", @@ -473,32 +434,22 @@ { "cell_type": "code", "execution_count": 9, - "id": "fcdd840a", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "metadata": {}, "outputs": [], "source": [ "results = {backend: {} for backend in backends}\n", "\n", "for backend in backends:\n", - " exps = experiments[backend]\n", + " exps = experiments_neq[backend]\n", " for i, exp in exps.items():\n", " results[backend][i] = experiment(\n", - " backend, exp['n_runs'], exp['n_ref'], exp['n_test'], exp['n_features']\n", + " 'mmd', backend, exp['n_runs'], exp['n_ref'], exp['n_test'], exp['n_features']\n", " )" ] }, { "cell_type": "markdown", - "id": "27307020", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, + "metadata": {}, "source": [ "The below plots illustrate that KeOps indeed still provides large speed ups over PyTorch. The x-axis shows the reference batch size $N_\\text{ref}$. Note that $N_\\text{ref} / N_\\text{test} = 10$." ] @@ -506,16 +457,11 @@ { "cell_type": "code", "execution_count": 10, - "id": "0a3c0d27", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAEWCAYAAAAZ9I+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd5wU9f3H8deHDtKLCAKCAQREUThRrNgIikKIAbEgtmA3sUbNL7aoMbG3WFA0qNgwKCqKGrCCCigIIigiCEc7QJAOd/f5/fGd0+W8zu7Nlffz8djH7s7Mznxmd+/ms99q7o6IiIhUblXiDkBERETip4RARERElBCIiIiIEgIRERFBCYGIiIighEBERERQQlDhmJmbWfsC1j9iZn9LeH6Bma0wsw1m1qQEx2sbHbNaSWNOlugc9iyF47SJjlW1FI7V3Mw+MLP1ZnZXqo8nRWdm15nZ43HHIZIsSghKyMwWmtk2M2uaa/kX0QWybfT8qej5gFzb3RMtPzN6fqaZZUUXmg1m9r2ZPWlmHZMZt7uf7+5/j45ZHbgb6OPudd19dWEJxc6IzvGjJO3rPTM7N3FZdA4LkrH/XMdaaGbHJBznh+hYWck+Vh6GA6uA+u5+xc7uzMxamNk4M1ua+D1NWF/TzEaa2U9mttzMLs+1/mgzm2tmm8xskpntsbMxxcHMbjSzZ4qxfW8zW5K4zN1vc/dz83uNSHmjhGDnfA+ckvPEzPYB6uSx3TfAGQnbVQMGA9/l2m6Ku9cFGgDHAJuB6WbWNRnB5vGLtjlQC/gqGfuXlNgDmOMlGEEsn1KbbOAt4KR8XnYj0CE67pHA1WbWN9pfU+C/wN+AxsA04IXixlVcZaH0SaRScHfdSnADFgL/B0xNWHYn8FfAgbbRsqei5SuARtGyE4A3gY+AM6NlZwIf5XGc14ExBcRxFbAMWAqcHR27fcKxHwbGAxsJScZTwC1Ax2iZAxuAicAH0fON0bKT8zhe1eh8VgELgIui11SL1jcAnohiSo+OVRXoDGwBsqJ9r422rxnt74foPXoEqJ1wvAHADOAnQgLVF7g12s+WaF8PRtsmnnsDYBSQASyKPqsqie91dNwfCYndcfm8v08TLqKbo2NdDbTNdc7vRec5OdrmNaAJ8GwU99Sc70O0fSfgHWANMA8YnM+xnwK2A9ui/R4TvV/3Rp/30uhxzWj73sAS4C/AcuDpAr431Uj4niYsX0ooMcp5/nfg+ejxcGBywrpdovelUwF/I9cCc6L3+UmgVsL6E6LPdm303u2b67V/Ab4EtkbxLiR8378kfEefICS1bwLrgXf55W+sN7Akj3iOIXyHtkXv7QZgZrT+LODraF8LgPNynWd2tP0GoCUheXomYf/9Ccn12ug70TnXsa+MYl9HSKRqReuaEv7O10bfiQ+Jvqu66Vaat9gDKK+3hH8u8wgXu6rRP+M9+HVCcAvwGHBBtOxFQslCURKCs4EV+cTQl3AR7Rr90xrNrxOCdcAhhNKgWjnxROvbknBhi5b9/Pp8jnk+MBdoTfiVOIkdL45jgUejeHYFPkv4x/qrcwTuAcZF+6pHuJj+I1rXM4r/2Cj+3YkuPtE/3HNz7Svx3EcBr0b7bEsopTknIY7twB+jz+0CwoXQCvqsE57v8L5FscwHfkNIROZExzuGcCEbBTwZbbsLsJhw8akG7E9Irrrkc+yfP6/o+c3AJ9F724xwIf17tK43kAn8k5A41M5rn9G2v0oIgEbRsuYJy/4AzIoe3wc8nGs/s4GTCnjfZid8Vz7ml+/e/sBK4MDoMxgWbV8z4bUzotfWTlj2CSEJ2D16/efRvmoRktobEt6LPBOC6PGNJFzMo2X9os/QgCOATUD3Avb38z74JcE+FqhOSBznAzUSjv0ZIZFoTEg8zo/W/YOQCFePboeRz3dRN91SeVOVwc57mlAdcCzhjzw9n+1GAWeYWUPCP5tXirj/pYR/IHkZTLjQzHb3jYR/ULm96u4fu3u2u28p4jELMhi4190Xu/sawj8zIDSAA44H/uzuG919JeGCPySvHZmZEX51Xubua9x9PXBbwvbnACPd/Z0o/nR3n1tYgFHVyBDgWndf7+4LgbuAoQmbLXL3ER7aAfwHaEG40JTUk+7+nbuvI/xi/c7d33X3TOAlwkULwq/ihe7+pLtnuvsXwMvAoCIe5zTgZndf6e4ZwE25ziubcFHc6u6bi3kOdaP7dQnL1hGSqpz169hR4vq8PJjwXbmVX6rYhgOPuvun7p7l7v8hlAQclPDa+6PXJp7HA+6+wt3TCb+kP3X3L6Lv9lh+eZ+Lzd3fiD5Dd/f3gbcJF+eiOBl4I/qubieUPtUGDs51Pkuj9+I1YL9o+XbC928Pd9/u7h+6uyaZkVKnurmd9zShqL0d4aKfJ3f/yMyaEaoUXnf3zeF6WKjdCcWIeWkJTE94viiPbRYX5SDF0DLXPhOPuQfhF86yhHOrUkAMzQhtLqYnbG+EX4wQfh2OL0GMTaM4EmNbRHgvcyzPeeDum6Lj16XkViQ83pzH85x97wEcaGZrE9ZXI3yPiqIlvz6vlgnPM3Yi8dsQ3dcnVMfkPF6fsL5+rtckrs9L7u9KTqx7AMPM7JKE9TXY8Vzy+t4U9X0uNjM7DriB8Gu/CuG7OauIL9/hc3H3bDNbTD7fOULpQ8653kFI5t+OvoePufvtJTgFkZ2iEoKd5O6LCHXQxxMaXBXkGeAKCkgc8jCQ8EsoL8sIF80cbfIKsRjHKoqCjrmY8Cuvqbs3jG713X3vfGJZRfgnvnfC9g08NKzM2d9v8omjoPNaRfjVldgCvg35l94UJpnv4WLg/YTzbeihx8IFRXz9Un59XksTnpc4Vnf/kfD5dktY3I1fGp1+lbjOzHYhfD4FNUrN/V3JiXUxcGuu96GOuz+XGFLJzgQIxfc/N/CNSo2a5bdvM6tJKKm5k1Bl0pCQjFpe2+dhh88lKv1qTRG+c1Ep1hXuviehHcLlZnZ0Ya8TSTYlBMlxDnBUVGxfkPsJVQsfFLSRmVU1s3Zm9gCh7vKmfDZ9ETjTzLqYWR3Cr5udtQIoqC//i8ClZtbKzBoB1+SscPdlhGLWu8ysvplVMbPfmNkRCftuZWY1ou2zgRHAPWa2K4CZ7W5mv422fwI4K+rqViVa16mwOKNqgBeBW82sXtQ17nJCQlYShb0nxfE60NHMhppZ9eh2gJl1LuLrnwP+z8yaRa3+r6eY52VmtQhtDABqRs9zjIr23yh6r/9IaMcAoUi+q5mdFL3meuDLQqpxLoq+K40JpWM5vRJGAOeb2YEW7GJm/cysoOqH4vgGqBXtszqhUWnNhPUrgLZmlvM/sEa0PgPIjEoL+uTavomZNcjneC8C/aLvanVC4r+V0MajQGZ2gpm1j5KIdYQGs9lFPVGRZFFCkARRveO0Imy3xt3/V0D9YC8z20Bomf4eoTj2AHfPs9jS3d8ktDKfSGjANLEk8edyI/AfM1trZoPzWD8CmADMJDToyl0qcgbhn2tOy/IxhPpRovi+Apab2apo2V+i2D8xs58ILcX3is7vM0Lju3sI/yjf55dfYfcBfzCzH83s/jzivITwK3EBofHmaGBk0d6CX/kH4SK51syuLOE+gPBrkHChGUL4VbmcXxoBFsUthO5+XxKKsz+PlhVHTo8JCA1EE+vobyD05lhEeL/vcPe3otgzCN0VbyV8tgeST/uQBKMJSeKCaL+3RPuaRkg2Hoz2NZ/Q2DMporYcFwKPE36lbyQ0+s3xUnS/2sw+jz6XSwkX9h+BUwmNXXP2N5eQjC2IvgeJVRu4+zzgdOABQgnVicCJ7r6tCOF2IHzvNwBTgH+7+6TinbHIzjO1XRGRVDCzhYSeIO/GHYuIFE4lBCIiIqKEQERERFRlICIiIqiEQERERKhgAxM1bdrU27ZtG3cYIiLlxvTp01e5e7PCt5SKrkIlBG3btmXatEJ7/4mISMTM8hrhVCohVRmIiIiIEgIRERFRQiAiIiIoIRARERGUEIiIiAhKCERERAQlBCIiIoISAhERESGFCYGZjTSzlWY2O5/1V5nZjOg228yyzKxxtG6hmc2K1pX7kYa+/x569YL33487EhERkbylsoTgKaBvfivd/Q5338/d9wOuBd539zUJmxwZrU9LYYylompV6N4dGjaMOxIREZG8pWzoYnf/wMzaFnHzU4DnUhVL3Nq0gYceijsKERGR/MXehsDM6hBKEl5OWOzA22Y23cyGF/L64WY2zcymZWRkpDLUEtu4EbZvjzsKERGR/MWeEAAnAh/nqi441N27A8cBF5nZ4fm92N0fc/c0d09r1qxsTtj1179CGQ1NREQEKBsJwRByVRe4e3p0vxIYC/SMIa6kSU+H3XaLOwoREZH8xZoQmFkD4Ajg1YRlu5hZvZzHQB8gz54K5cXSpbD77nFHISIikr+UNSo0s+eA3kBTM1sC3ABUB3D3R6LNBgJvu/vGhJc2B8aaWU58o939rVTFWRrS0+Gww+KOQkREJH+p7GVwShG2eYrQPTFx2QKgW2qiKn3Z2SohEBGRsi9lCYEEmZlw881hYCIREZGySglBitWoAddcE3cUIiIiBSsLvQwqtLVrYdEiyMqKOxIREZH8KSFIsRdfhLZtQzsCERGRskoJQYotXQpmGodARETKNiUEKZaeDrvuCtWrxx2JiIhI/pQQpJi6HIqISHmghCDF0tOhZcu4oxARESmYuh2m2PXXwy67xB2FiIhIwZQQpNjvfx93BCIiIoVTlUEKbdgAH30EP/0UdyQiUlG98QZcdRW4xx2JlHdKCFJo5swwqdHkyXFHIiIVjTvceSeceCL873+wcWPhrxEpiBKCFEpPD/fqZSAiybR1K5x9digZOOkk+PBDqFs37qikvFNCkEJKCEQk2VauhKOOgqeeghtugBdeUMNlSQ41Kkyh9HSoVQsaNYo7EhGpCL78MlQRZGSERGDw4LgjkopEJQQplJ4eSgfM4o5ERMq7V16Bgw8OU6p/+KGSAUk+lRCk0F/+EjJ5EZGScofbb4frroMDDgiJgQY7k1RQQpBC++0XdwQiUp5t2QLnngvPPgunnAJPPAG1a8cdlVRUqjJIEfdQx/fdd3FHIiLl0bJlcMQRIRm49dZwr2RAUillCYGZjTSzlWY2O5/1vc1snZnNiG7XJ6zra2bzzGy+mV2TqhhTae1aGDIEXn017khEpLz5/PNQPfDVV/Df/4bqArVFklRLZQnBU0DfQrb50N33i243A5hZVeAh4DigC3CKmXVJYZwpoS6HIlISL70Ehx4KVarAxx/DwIFxRySVRcoSAnf/AFhTgpf2BOa7+wJ33wY8DwxIanClYOnScK/GPyJSFO5w002h98D++8PUqdCtW9xRSWUSdxuCXmY208zeNLO9o2W7A4sTtlkSLcuTmQ03s2lmNi2jDDXpVwmBiBTVpk2hivHGG+GMM2DiRGjePO6opLKJMyH4HNjD3bsBDwCvlGQn7v6Yu6e5e1qzZs2SGuDOyEkIVEIgIgVZsiTMefLSS3DHHWEEwpo1445KKqPYEgJ3/8ndN0SPxwPVzawpkA60Tti0VbSsXPnjH8NMh7VqxR2JiJRVn30GPXvCN9/AuHFw5ZVqPCjxiS0hMLPdzMJX38x6RrGsBqYCHcysnZnVAIYA4+KKs6SaN4dDDok7ChEpq0aPhsMPDz8apkyBE06IOyKp7FI2MJGZPQf0Bpqa2RLgBqA6gLs/AvwBuMDMMoHNwBB3dyDTzC4GJgBVgZHu/lWq4kyVUaOgTRvo3TvuSESkLMnOhr/9DW67LSQEL78MTZvGHZUIWLgGVwxpaWk+bdq0uMMAoEUL6NcPHn887khEpKzYsAGGDg3DD597Ljz0ENSoEW9MZjbd3dPijULKAg1dnALbt8OKFephICK/WLQI+veH2bPh3nvh0kvVXkDKFiUEKbBiRehTrB4GIgIweXIYYGjrVhg/Hn7727gjEvm1uMchqJA0BoGI5PjPf+DII6F+ffjkEyUDUnYpIUiBnFEKlRCIVF5ZWXD11XDmmWGcgU8/hU6d4o5KJH+qMkiBfv1gwQJVGYhUVj/9BKeeCm+8ARdeGNoMVK8ed1QiBVNCkAI1akC7dnFHISJxWLAgNB6cOzf0IrjwwrgjEikaJQQp8OyzsHEjDB8edyQiUprefx9OOimMNfD223DUUXFHJFJ0akOQAk8+GcYjF5HKY8QIOOaYMMjQp58qGZDyRwlBCqSnq/2ASGWRmQl//nMoETz66NCToEOHuKMSKT4lBCmQnq4eBiKVwdq1oRHxffeFpOD116Fhw7ijEikZtSFIsvXrw00JgUjF9u23cOKJoRHhiBFhKGKR8kwJQZKtWBHuVWUgUnG9+y4MHgxVqoTHhx8ed0QiO08JQZK1bw9btsQdhYikyr//HeYh6NQJXntNXYyl4lAbghSoWTPcRKTi2L49jClw0UVw3HFhfgIlA1KRKCFIsldegUsuCcOWikjFsGYN9O0LDz8chiN+5ZUwN4FIRaKEIMkmToRRo6Bq1bgjEZFk+PprOPBA+OijML7IP/+pv2+pmNSGIMnU5VCk4njrLTj5ZKhVCyZNgoMPjjsikdRRCUGSLV2qHgYi5Z17mJCoX7/QTmDqVCUDUvEpIUgylRCIlG/btoVRBy+7DAYMCFUFbdrEHZVI6qUsITCzkWa20sxm57P+NDP70sxmmdlkM+uWsG5htHyGmU1LVYzJ5h7uW7eONw4RKZmMjDAfweOPw//9H4wZA3Xrxh2VSOlIZRuCp4AHgVH5rP8eOMLdfzSz44DHgAMT1h/p7qtSGF/SmcEPP/ySGIhI+TF7dhh5cNkyGD0aTjkl7ohESlfKSgjc/QNgTQHrJ7v7j9HTT4BWqYqltJnFHYGIFMfrr0OvXrB1K3zwgZIBqZzKShuCc4A3E5478LaZTTez4QW90MyGm9k0M5uWkZGR0iAL89FHMHAgLFoUaxgiUkTucMcd0L8/7LVXaDzYs2fcUYnEI/aEwMyOJCQEf0lYfKi7dweOAy4ys3xHCnf3x9w9zd3TmjVrluJoCzZ7dhiwpJo6c4qUeVu2wJlnhoGGBg0KJQNqECyVWawJgZntCzwODHD31TnL3T09ul8JjAXKRc6enh4mO2nePO5IRKQgK1bAUUeFQcRuvhmefx7q1Ik7KpF4xfZb1szaAP8Fhrr7NwnLdwGquPv66HEf4OaYwiyWpUtDMqASApGya8aMUEWwahW89BL84Q9xRyRSNqTs0mVmzwG9gaZmtgS4AagO4O6PANcDTYB/W2iFl+nuaUBzYGy0rBow2t3fSlWcyaQxCETKtv/+F4YOhcaNQ5uf7t3jjkik7EhZQuDuBbbTdfdzgXPzWL4A6PbrV5R9jRpBqwrTV0Kk4nCH224LYwsceCCMHQstWsQdlUjZosLtJHruubgjEJHcNm+Gc84Jf5+nnRYGHapVK+6oRMqe2HsZiIikytKlcMQRodHgP/4BTz+tZEAkP0oIkmTRIkhLg//9L+5IRARg2jQ44ACYMydUEVxzjQYNEymIEoIk+eEHmD4dsrLijkREXngBDjsMqleHyZPDJEUiUjAlBEmSnh7u1ctAJD7Z2XDDDTBkCPToAZ99BvvuG3dUIuWDGhUmiRICkXht3AjDhsHLL8NZZ8HDD0PNmnFHJVJ+KCFIkvT0MNJZgwZxRyJS+SxeHKoFZsyAO++Eyy9XewGR4lJCkCS77w7HHad/QiKl7ZNP4He/g02bwqyFxx8fd0Qi5ZMSgiS54oq4IxCpfJ55Bs49NyTkEydCly5xRyRSfqlRoYiUO9nZcO21YRjiXr1C40ElAyI7RwlBErhDu3Zw771xRyJS8a1fDwMHwu23w/DhMGECNGkSd1Qi5Z+qDJJgzRpYuFDtB0RSbeHCMFPhV1/B/ffDxRfr704kWZQQJMHSpeG+Zct44xCpyD76CH7/e9i2Dd58E/r0iTsikYpFVQZJoDEIRFLrySfhqKPCjKKffqpkQCQVlBAkQU5CoBICkeTKygo9eM4+O0xS9MknsNdecUclUjGpyiAJdt89NHLS/OoiybNuHZxySqgeuOQSuPtuqKb/WCIpU+ifl5n1Ak4HDgNaAJuB2cAbwDPuvi6lEZYDffuGm4gkx3ffwYknwrffwiOPwHnnxR2RSMVXYEJgZm8CS4FXgVuBlUAtoCNwJPCqmd3t7uNSHWhZlp0NVVT5IpIU770HJ50UHr/9Nhx5ZKzhiFQahZUQDHX3VbmWbQA+j253mVnTlERWjhxwQKjXHD067khEyrfHHoOLLoIOHWDcOGjfPu6IRCqPAn/X5iQDZraLmVWJHnc0s/5mVj1xm7yY2UgzW2lms/NZb2Z2v5nNN7Mvzax7wrphZvZtdBtWkpMrLUuWQN26cUchUn5lZsKll4aqgWOPhSlTlAyIlLaiFnR/ANQys92Bt4GhwFNFeN1TQEG168cBHaLbcOBhADNrDNwAHAj0BG4ws0ZFjLVUbd8OK1eqy6FISf34Y5iQ6IEHQo+C117TrKEicShqQmDuvgn4PfBvdx8E7F3Yi9z9A2BNAZsMAEZ58AnQ0MxaAL8F3nH3Ne7+I/AOBScWsVm2LNyry6FI8X3zDRx0UGg38MQTYeriqlXjjkqkcipyQhD1NjiN0LsAIBl/trsDixOeL4mW5bc8r8CGm9k0M5uWkZGRhJCKR4MSiZTMO+/AgQeGob8nTgxjDYhIfIqaEPwJuBYY6+5fmdmewKTUhVV07v6Yu6e5e1qzZs1K/fhNmoS6z86dS/3QIuWSOzz4IBx3HLRqBVOnwqGHxh2ViBRpmI+o6P+DhOcLgEuTcPx0oHXC81bRsnSgd67l7yXheEnXsSPcd1/cUYiUD9u3h0GGHn00TFL0zDNQr17cUYkIFFJCYGYjzGyffNbtYmZnm9lpO3H8ccAZUW+Dg4B17r4MmAD0MbNGUWPCPtGyMmftWti6Ne4oRMq+1avDHASPPgrXXANjxyoZEClLCisheAj4W5QUzAYyCAMTdQDqAyOBZ/N7sZk9R/il39TMlhB6DuR0V3wEGA8cD8wHNgFnRevWmNnfganRrm5294IaJ8bm4ovh44/h++/jjkSk7JozJ4w8mJ4OTz8Np58ed0QikluBCYG7zwAGm1ldII1fhi7+2t3nFbZzdz+lkPUOXJTPupGEhKNMW7pUDQpFCjJ+PAwZAnXqhN4EBx0Ud0QikpeitiHYQBmtw49bejp06xZ3FCJljzvccw9cdRXsu28YebB168JfJyLx0Aj8O8E9JAQqIRDZ0datcM45YaChgQPho4+UDIiUdUoIdsL69bBxowYlEkm0ciUccww8+ST87W/w4ouwyy5xRyUihSnW7OJmVicasVAAM7jjDs3GJpJj1qzQeHDFCnj+eTj55LgjEpGiKlIJgZkdbGZzgLnR825m9u+URlYO1KsHV14JPXrEHYlI/MaNg4MPDmMNfPihkgGR8qaoVQb3EOYXWA3g7jOBw1MVVHmxciV89x1kZ8cdiUh83OH22+F3vwsjdk6dCmlpcUclIsVV5DYE7r4416KsJMdS7jzxRJiidcuWuCMRiceWLXDGGXDttaFE4P331aZGpLwqakKw2MwOBtzMqpvZlcDXKYyrXEhPh4YNQ/9qkcpm+XLo3TsMP/z3v8Po0VC7dtxRiUhJFbVR4fnAfYQZB9OBt8lnQKHKRIMSSWX1xRdhLoI1a+Dll+H3v487IhHZWUUdmGgVYepjSaAxCKQyevnlUE3QpEkYtnu//eKOSESSoUgJgZm1Ay4B2ia+xt37pyas8iE9HfbeO+4oREqHe6gauOGGMPzw2LGw225xRyUiyVLUKoNXgCeA1wC1qY/cc48aUEnlsGkTnH02vPBCKB149FGoVSvuqEQkmYqaEGxx9/tTGkk5NGhQ3BGIpF56OgwYAJ9/Dv/8Z5ibwCzuqEQk2YqaENxnZjcQGhNuzVno7p+nJKpyYPVqmD0bunfXnO5ScU2dGpKB9evh1VfDKIQiUjEVNSHYBxgKHMUvVQYePa+UPvooDMSiQVikonruuVBNsNtuMGEC7LNP3BGJSCoVNSEYBOzp7ttSGUx5kp4e7tXLQCqa7OzQcPCWW+Cww0KvgmbN4o5KRFKtqAMTzQYapjKQ8mbpUqhaFXbdNe5IRJJn48bQNuaWW8L0xe++q2RApLIoaglBQ2CumU1lxzYElbbbYXp6KEqtWjXuSESS44cfwmBDs2aFHjR/+pMaD4pUJkVNCG5IaRTlkAYlkopkyhQYOBA2b4Y33oC+feOOSERKW1FHKny/JDs3s76EIY+rAo+7++251t8DHBk9rQPs6u4No3VZwKxo3Q9lrTTi9tvDP0+R8m7UKPjjH6F1a5g0KcxYKCKVT4EJgZl95O6Hmtl6Qq+Cn1cB7u71C3htVeAh4FhgCTDVzMa5+5ycbdz9soTtLwH2T9jFZncvs4Oidu8edwQiOycrC667Dv71LzjqKHjxxTAcsYhUToU1KtwFwN3ruXv9hFu9gpKBSE9gvrsviHonPA8MKGD7U4Dnihx5jLZuhWefhYUL445EpGTWrw9VBP/6F5x/Prz1lpIBkcqusITAC1lfkN2BxQnPl0TLfsXM9gDaARMTFtcys2lm9omZ/S6/g5jZ8Gi7aRkZGTsRbtEtXgynnx7mfhcpb77/Hg4+GMaPh4cegocfhurV445KROJWWBuCXc3s8vxWuvvdSYpjCDDG3bMSlu3h7ulmticw0cxmuft3ecTwGPAYQFpa2s4kMEWmMQikvPrgAzjpJMjMDKUCxxwTd0QiUlYUVkJQFagL1MvnVpB0oHXC81bRsrwMIVd1gbunR/cLgPfYsX1BrJQQSHn0xBMhAWjSBD77TMmAiOyosBKCZe5+cwn3PRXoEE2dnE646J+aeyMz6wQ0AqYkLGsEbHL3rWbWFDgE+FcJ40g6JQRSnmRmhgmJ7r0X+vQJMxY21DBjIpJLYQlBiYclcfdMM7sYmEAoaRjp7l+Z2c3ANHcfF206BHje3ROL+zsDj5pZNqEU4/bE3glxS0+HunWhfmHNKkVitm4dDBkSqgcuvRTuuguqFXX0ERGpVGzH63Culf/muWUAACAASURBVGaN3X1NKcazU9LS0nzatGkpP86KFSEpUNdDKcvmzw+zE86fHxoPDh8ed0RSFpnZdHfXFG1ScAlBeUoGSlPz5uEmUlZNnAh/+ANUqRLmIzjiiLgjEpGyrqiTG0mCf/8bPv447ihE8vbww6GtQIsWofGgkgERKQolBMWUnQ1//jO89lrckYjsaPt2uPhiuPDCMBfBlCmw555xRyUi5YUSgmJavTr8423ZMu5IRH6xZg0cd1xoK3DllfDqq2r0KiLFo/bGxaQuh1LWzJ0bGg/+8AM8+SSceWbcEYlIeaSEoJiWLg33SgikLJgwAU4+GWrUCA0JDzkk7ohEpLxSlUEx5ZQQqMpA4uQO990Hxx8Pe+wBU6cqGRCRnaOEoJjOPDMUzaqEQOKybRucd15o3Nq/f+jxsscecUclIuWdEoJiql4dWreGqlXjjkQqo1WrQpfCESPguuvg5ZfDqJkiIjtLbQiK6eGHoVYtOOusuCORyuarr0LjwaVL4dln4dRfzQwiIlJyKiEopkcfhf/+N+4opLJ5/XXo1Qs2b4b331cyICLJp4SgmNLT1X5ASo873HlnaCvQoUNoPHjggXFHJSIVkRKCYti6NdThKiGQ0rB1a6iauuqqMC/Bhx9Cq1ZxRyUiFZUSgmJYtizcq8uhpNrKlXDUUfCf/8CNN8Lzz0OdOnFHJSIVmRoVFsPKlWH2OJUQSCrNnBmqCDIy4MUXYdCguCMSkcpACUEx9OwZinFFUuWVV+D006Fhw1BF0KNH3BGJSGWhKoNiqlYt3ESSyR1uuw0GDoS99w6NB5UMiEhpUkJQDE89BZddFncUUtFs3hxKBf7619Cd8L33oEWLuKMSkcpGCUExvP02jBsXdxRSkSxbBr17w+jRoYTgmWegdu24oxKRyiilCYGZ9TWzeWY238yuyWP9mWaWYWYzotu5CeuGmdm30W1YKuMsqqVL1cNAkmf6dDjggDAC4dixcO21YBZ3VCJSWaWsNtzMqgIPAccCS4CpZjbO3efk2vQFd78412sbAzcAaYAD06PX/piqeIsiPV31upIcL70Ew4ZBs2ZhcqJu3eKOSEQqu1SWEPQE5rv7AnffBjwPDCjia38LvOPua6Ik4B2gb4riLBL3UEKgLoeyM7Kz4aabYPBg6N49NB5UMiAiZUEqE4LdgcUJz5dEy3I7ycy+NLMxZta6mK/FzIab2TQzm5aRkZGMuPO0aRM0ahRmOhQpiU2bYMiQMNDQsGHwv//BrrvGHZWISBB3B7rXgOfcfauZnQf8BziqODtw98eAxwDS0tI8+SEGu+wCS5akau9S0S1ZAgMGwBdfwB13wBVXqL2AiJQtqSwhSAcSf0+3ipb9zN1Xu3vOUD+PAz2K+lqR8uLTT0PjwW+/hddegyuvVDIgImVPKhOCqUAHM2tnZjWAIcAOnfbMLLG3dX/g6+jxBKCPmTUys0ZAn2hZbF5/PcxFv3p1nFFIeTN6NBxxROhKOGUK9OsXd0QiInlLWZWBu2ea2cWEC3lVYKS7f2VmNwPT3H0ccKmZ9QcygTXAmdFr15jZ3wlJBcDN7r4mVbEWxcyZISnQBDNSFNnZ8Le/hbEFjjgCxoyBpk3jjkpEJH8pbUPg7uOB8bmWXZ/w+Frg2nxeOxIYmcr4iiM9HRo31qAxUjD3MNLgP/4B77wDf/wjPPgg1KgRd2QiIgWLu1FhuTF7NrRvH3cUUlZt3AjPPgsPPBC+K02ahETgwgvVXkBEygcNXVwE27fDtGnQq1fckUhZ8/33oZFgq1Zw3nlh4qsnnoDFi+Gii5QMiEj5oRKCIsjIgLS0UBcs4g4TJ8L994deA1WqwEknwSWXwCGHKAkQkfJJCUERtGwJH3wQdxQStw0b4OmnQ1XAnDlh2OHrroPzzw8lBCIi5ZkSgiLIyoKqVeOOQuLy3Xfw0EMwciSsWxfms3jqKTj5ZKhVK+7oRCqG6dOn71qtWrXHga6oOjsVsoHZmZmZ5/bo0WNlXhsoISiCzp1h0CC49da4I5HSkp0N774bqgXGjw8J4aBBoVrgoINULSCSbNWqVXt8t91269ysWbMfq1SpkrJRZyur7Oxsy8jI6LJ8+fLHCeP+/IoSgkKkp4cR5po1izsSKQ3r18N//hOqBebNg+bNw3gC552nqa9FUqyrkoHUqVKlijdr1mzd8uXLu+a3jRKCQkyZEu4PPjjeOCS1vv02JAFPPhmSgp49Q3uBQYOgZs24oxOpFKooGUit6P3NtzpGCUEhpkwJ9cT77Rd3JJJs2dkwYUKoFnjrLahePUxLfMklcOCBcUcnIlK61HCjEJMnhy6HGmmu4li3Du67Dzp1guOPhxkz4Kab4Icf4JlnlAyIVEbz5s2r0aFDh73jjiM//fv3b9e2bduuHTp02HvQoEFtt27dmvSWTEoICjFkCAwfHncUkgxz58LFF4cugn/+c5hbYPRoWLQIrr8edtst7ghFRPJ22mmnrVmwYMHsefPmfbVlyxa79957kz47ihKCQvzpTzB0aNxRSEllZ4dJqX7729BbZMQI+P3vYerUUPpzyikq/RGRHc2ZM6dG586du7z//vt1MjMzOe+881p17dq1c8eOHbvccccdTQGys7M577zzWnXo0GHvjh07dhkxYkQjgNdff71eWlraXr17927ftm3brqeeemqbrKwsMjMzOemkk9rmbH/TTTftWpyYTj755HVVqlShSpUqpKWlbVyyZEnS/3OpDUEBvvsO6tWDXYv1sUlZsHZtGDfgoYdgwYLQQ+CWW8JkQ/o8Rcq2s8+m9ezZJHVu2a5d2TRyJIsL227mzJk1hwwZ8puRI0d+36tXr8133nln0wYNGmTNnj37682bN9sBBxzQ6cQTT/zpk08+qTNr1qzaX3/99VfLli2r1rNnz859+vTZADBr1qxdvvjii9kdO3bcdvjhh3cYNWpUo/bt229dtmxZ9W+//fYrgFWrVpVodJutW7faCy+80OTuu+8u9FyKSwlBAa6+OtQvf/dd3JFIUc2ZEyYYGjUKNm2CQw8NMw8OHBgaDYqI5GfNmjXVfve737UfM2bMdz169NgC8O6779afO3dunXHjxjUCWL9+fdU5c+bU+vDDD+sNHjx4TbVq1WjdunXmgQceuOGjjz6q06BBg+x99tlnY5cuXbYBDB48eM2HH35Y94QTTvhp8eLFNYcNG9b6xBNPXDdw4MCfShLjsGHD2hx00EEb+vbtuyF5Zx4oIciHeyhSPvrouCORwmRlhWqBBx6A//0vdBM89dTQW2D//eOOTkSKqyi/5FOhXr16WS1bttw2adKkujkJgbvbXXfd9cNJJ520wwX8jTfeaJDffizXyGVmRrNmzbJmz549Z+zYsfUfeeSRZi+88ELjl156aWHONpmZmXTt2rULQN++fdfee++9S3Pv94orrmixatWqahMmTEjJz1S1IcjHokWwfLnGHyjL1qyBO+4I01L/7ndhIKHbboMlS0J1gZIBESmO6tWr+5tvvvndc8891+SRRx5pDHDssceue/jhh5vltOr/8ssva/70009VDj/88PVjxoxpnJmZydKlS6t99tlndQ877LCNEKoM5s6dWyMrK4sxY8Y0Puyww9YvW7asWlZWFmeeeebaf/zjH+mzZs3aoUqkWrVqzJ07d87cuXPn5JUM3H333U0nTpzY4JVXXllQNUVj6auEIB+TJ4d7TXlc9syaFUoDnnkGNm+Gww+HO++EAQPC9MMiIiVVv3797AkTJszv3bt3x3r16mVddtllqxYuXFhzn3326ezu1rhx4+3jx4//bujQoWsnT55ct3Pnznubmd90001L2rRpk/nll1/StWvXjeeff36bhQsX1jr44IN/Gjp06NrPPvus9jnnnNM2OzvbAG6++eYlxYnr6quv3qNFixZb09LSOgOccMIJP955553Lknnu+veZjylTYJddYJ994o5EADIzYdy4kAi8914YLOr000M3wm7d4o5ORMq7vfbaa1tOg7+mTZtmzZ49++ucdQ8++GA6kJ77NY8++ugS4FcX9nr16mVNmjRpfuKyXr16bZ4zZ87XubctqszMzOklfW1RKSHIx5//DH376hdn3Favhscfh3//OwwctMce8M9/wjnnQJMmcUcnIlJxpPRyZ2Z9gfuAqsDj7n57rvWXA+cCmUAGcLa7L4rWZQGzok1/cPc8Z2dKld/8JtwkHjNmhNKA0aNhyxY48sgwuuCJJ2oqahEpu0444YT1J5xwwvq44yiJlCUEZlYVeAg4llCkMtXMxrn7nITNvgDS3H2TmV0A/As4OVq32d1jmUHg669DG4LBg8M4BFI6tm+HV14JicCHH0KdOjBsWKgW6Jrv/FwiIpIMqexl0BOY7+4L3H0b8DwwIHEDd5/k7puip58ArVIYT5G98gqce264QEnqZWSE3gHt2oUkbMmS0EhwyRJ45BElAyIipSGVVQa7ww59SZcABU0bcw7wZsLzWmY2jVCdcLu7v5LXi8xsODAcoE2bNjsVcI7Jk8PEN40bJ2V3ko/p00NpwPPPw9atcMwxoa1Av36qFhARKW1losmcmZ0OpAFHJCzew93TzWxPYKKZzXL3Xw3G4O6PAY8BpKWl7fRc2u6hh0H/Um2xUHls3w4vvxwSgcmTQ0+Oc84J1QKdO8cdnYhI5ZXKKoN0oHXC81bk0W3DzI4B/gr0d/etOcvdPT26XwC8B5TKMDPffhtatmtAouRasQL+/ndo2zZMKLRiBdxzD6Snh/kGlAyISHlzzTXXJG2O1Msvv7zl9ddf37ykr588eXLt/fbbr1P79u13mGypOFKZEEwFOphZOzOrAQwBxiVuYGb7A48SkoGVCcsbmVnN6HFT4BAgsTFiysyYEe41IFFyTJ0KZ5wBbdqEKYb32ScMM/zNN6FrZ4N8B/8UESnb7r///hbF2T47O5usrKyUxFK3bt3sp59++vv58+d/9fbbb3973XXXtS7uBEopSwjcPRO4GJgAfA286O5fmdnNZpZTIH8HUBd4ycxmmFlOwtAZmGZmM4FJhDYEpZIQDB4cfr3qF2vJbdsWugsedBD07Aljx8J558HcufDWW6GNQBUNmi0iZci8efNqtGvXbu/+/fu323PPPffu27fvnuvXr68ybty4esccc8zPndDHjh1b/9hjj/3NhRdeuPvWrVurdOrUqUv//v3bAdx4443NO3TosHeHDh32vvnmm3fN2W/btm27Dhw4sG3Hjh33/u6772qMGTOmfpcuXTrvtddeXXr16tUxZ99ff/117Z49e+7VqlWrfW655ZZizcu67777bt1nn322ArRt23Z748aNM5ctW1asZgEpbUPg7uOB8bmWXZ/w+Jh8XjcZiG2MQE2PWzLLlsGjj4bb8uXQsSPcf3/oOli/ftzRiUh50rMne+Ve9vvfs+aaa8hYv54qRx9Nh9zrTz+dVZdeyuply6g2YAA7jCTz2WfMK+yYCxcurPXoo48u7NOnz8ZBgwa1veOOO5rdeOONK/70pz+1Wbp0abWWLVtmjhw5sslZZ5216tRTT1331FNP7Tp37tw5AB9++GGd0aNHN5k+ffrX7k6PHj06H3300eubNm2a9cMPP9R84oknvj/66KMXLl26tNrFF1/c9r333pvbqVOnbStWrPj5V/z8+fNrTZ48ed7atWurdu7cuetVV12VUbNmzWK3jZs0aVKd7du3W5cuXbYWvvUv9Dstwbp1YZrcnHkMpHDu8MkncNppYRTBm26C7t3hzTfDeA6XXKJkQETKh912221bnz59NgIMHTp09eTJk+tWqVKFwYMHrx4xYkTjVatWVf3888/rDho0aF3u17733nt1jz/++LX169fPbtCgQXa/fv1+nDRpUj2AFi1abDv66KM3Rtvt0rNnz/WdOnXaBtC8efOf6xD69Omztnbt2t6iRYvMxo0bb1+yZEmxf7QvWrSo+llnnbXniBEjFhZ3EqQy0cugrPj00zAGwYUXxh1J2bd1K7z4YigBmDYtXPQvvBAuugg6/CpvFxEpnoJ+0derR3ZB61u0ILMoJQK55TVtMcAFF1ywul+/fu1r1arlJ5544o/Vq1cv1n7r1KmTXZTtEksDqlatSmZm5g4BjRo1quFtt93WEuCxxx5bePjhh29KXL9mzZoqxx13XPsbbrghPScBKQ6VECSYMgXM4MCCRkuo5NLT4W9/C40EzzgDNmwIvQSWLIF771UyICLl17Jly2q8++67uwA8++yzjQ8++OANEOrkmzdvvv2uu+5qMXz48FU521erVs1zpkU+8sgjN4wfP77h+vXrq/z0009Vxo8f3+jII4/81RDGvXv33vjZZ5/Vmzt3bg2AxCqDwpxxxhlrc6ZIzp0MbNmyxfr169d+yJAhq88666wfS3L+SggSTJ4cRsVTEfeO3OHjj2HIkNBt8NZbQ9L09tswZ04oGdAQzyJS3rVt23bLAw88sOuee+6599q1a6tdeeWVGTnrhgwZsrpFixbbunfvviVn2WmnnZbRuXPnLv3792936KGHbjr11FNXd+/evXOPHj06Dx06NOOQQw7ZnPsYLVu2zLz//vsXDhw4sP1ee+3VZeDAgXsmI/aRI0c2mjp1at3Ro0c37dSpU5dOnTp1mTx5cu3i7MPcd3osnzIjLS3Np02bVqLXZmdDo0ahj/wjjyQ5sHJqy5YwiuD998MXX4QuguecE6oF9kzKV1hE4mZm0909Le44Zs6cubBbt26rCt8yNebNm1fjhBNO6JAzBXJuZ5xxRpv9999/02WXXRZbjMkwc+bMpt26dWub1zq1IYisWhW6Gh5+eNyRxG/xYnj4YRgxIrwvXbqEJOn008PIgiIilcnee+/duXbt2tmPPvro4sK3Lr+UEER23TW0lq+s3OGjj0JpwNix4Xn//qGXwJFHhrYVIiIV1V577bUtv9KBr7766uvSjicOSggi7pXzord5cxhE6IEHYObMUG1y+eWhXUDbtnFHJyKVSHZ2drZVqVKl4tRjlzHZ2dkG5NvjQY0KI/vvD9dcE3cUpWfRIvjLX6BVqzDVc3Y2PPZY6C3wr38pGRCRUjc7IyOjQXTRkiTLzs62jIyMBsDs/LZRCQFhMqOZM0Mr+orMHd5/P1QLvPpqWDZwYKgWOPzwyllCIiJlQ2Zm5rnLly9/fPny5V3Rj9VUyAZmZ2ZmnpvfBkoI+KXtQEWd0GjjRnj22VAtMHs2NGkCV18NF1wQxhMQEYlbjx49VgKaeD5GSggI4w9UrQoHHBB3JDtv+/YwNsDnn/9ymzEDNm2C/faDJ54IXStrF6t3qoiIVHRKCAgjFO6/P9SpE3ckxbN5M8yaFS76X3wR7r/8Msw2CFC3bjivc8+FQYPgkENULSAiInlTQgCceGLZ71+/fn1o55D4y3/OHMiZWrtRozCp0J/+FO67d4f27TXNsIiIFI0SAuCyy+KOYEdr1vzyiz/n/ptvQqNAgObNoUePME5AzsV/jz30619EREpOCUHMVqzY8Vf/55/DwoW/rG/TJlzwTzvtl4t/ixaxhSsiIhWUEoJS4h6GBE688H/xBSxd+ss2HTpAz55w/vnhwr///tC0aXwxi4hI5aGEIMnWrIH58+Hbb8N9zuNvvoEfowkpq1QJ8yYcffQvv/q7dQuTB4mIiMRBCUEJrF6940U/8eK/Zs0v25lB69ahcd/gwbDvvuHiv+++5a9Hg4iIVGwpTQjMrC9wH1AVeNzdb8+1viYwCugBrAZOdveF0bprgXOALOBSd5+QylgBMjMhIwOWLYPly8Mt5/GyZfDDD+Gin/NLP8QZ6vlzLvrt24ei//btwxTBtWqlOmoREZGdl7KEwMyqAg8BxwJLgKlmNs7d5yRsdg7wo7u3N7MhwD+Bk82sCzAE2BtoCbxrZh3dPSvZcbpDWhqkp4dkIDuPaR8aNoTddgvj/g8ZsuNFv107XfRFRKT8S2UJQU9gvrsvADCz54EBQGJCMAC4MXo8BnjQzCxa/ry7bwW+N7P50f6mJDtIM+jSJXTj22230II/8b55c43qJyIiFV8qE4LdgcUJz5cAB+a3jbtnmtk6oEm0/JNcr909r4OY2XBgOECbEg7M//TTJXqZiIhIhVHux7Fz98fcPc3d05o1axZ3OCIiIuVSKhOCdKB1wvNW0bI8tzGzakADQuPCorxWREREkiSVCcFUoIOZtTOzGoRGguNybTMOGBY9/gMw0d09Wj7EzGqaWTugA/BZCmMVERGp1FLWhiBqE3AxMIHQ7XCku39lZjcD09x9HPAE8HTUaHANIWkg2u5FQgPETOCiVPQwEBERkcA8Z8acCiAtLc2nTZsWdxgiIuWGmU1397S445D4lftGhSIiIrLzlBCIiIiIEgIRERGpYG0IzCwDWFTIZk2BVaUQTlmj865cdN6Vy86c9x7urkFcpGIlBEVhZtMqYwManXflovOuXCrreUtyqcpARERElBCIiIhI5UwIHos7gJjovCsXnXflUlnPW5Ko0rUhEBERkV+rjCUEIiIikosSAhEREak8CYGZ9TWzeWY238yuiTuekjCz1mY2yczmmNlXZvanaHljM3vHzL6N7htFy83M7o/O+Usz656wr2HR9t+a2bCE5T3MbFb0mvvNzEr/TPNmZlXN7Aszez163s7MPo1ifSGaVZNolswXouWfmlnbhH1cGy2fZ2a/TVheJr8fZtbQzMaY2Vwz+9rMelWGz9vMLou+47PN7Dkzq1URP28zG2lmK81sdsKylH+++R1DKjl3r/A3wmyL3wF7AjWAmUCXuOMqwXm0ALpHj+sB3wBdgH8B10TLrwH+GT0+HngTMOAg4NNoeWNgQXTfKHrcKFr3WbStRa89Lu7zTjj/y4HRwOvR8xeBIdHjR4ALoscXAo9Ej4cAL0SPu0SffU2gXfSdqFqWvx/Af4Bzo8c1gIYV/fMGdge+B2onfM5nVsTPGzgc6A7MTliW8s83v2PoVrlvlaWEoCcw390XuPs24HlgQMwxFZu7L3P3z6PH64GvCf88BxAuHET3v4seDwBGefAJ0NDMWgC/Bd5x9zXu/iPwDtA3Wlff3T9xdwdGJewrVmbWCugHPB49N+AoYEy0Se7zznk/xgBHR9sPAJ53963u/j0wn/DdKJPfDzNrQLhgPAHg7tvcfS2V4PMmTM1e28yqAXWAZVTAz9vdPyBM/Z6oND7f/I4hlVhlSQh2BxYnPF8SLSu3omLR/YFPgebuvixatRxoHj3O77wLWr4kj+Vlwb3A1UB29LwJsNbdM6PnibH+fH7R+nXR9sV9P+LWDsgAnoyqSh43s12o4J+3u6cDdwI/EBKBdcB0Kv7nnaM0Pt/8jiGVWGVJCCoUM6sLvAz82d1/SlwX/RKoUH1JzewEYKW7T487llJWjVCc/LC77w9sJBTv/qyCft6NCL9g2wEtgV2AvrEGFZPS+Hwr4ndISqayJATpQOuE562iZeWOmVUnJAPPuvt/o8UrouJBovuV0fL8zrug5a3yWB63Q4D+ZraQULx7FHAfoci0WrRNYqw/n1+0vgGwmuK/H3FbAixx90+j52MICUJF/7yPAb539wx33w78l/AdqOifd47S+HzzO4ZUYpUlIZgKdIhaKdcgNDwaF3NMxRbViz4BfO3udyesGgfktCweBryasPyMqHXyQcC6qJhwAtDHzBpFv8b6ABOidT+Z2UHRsc5I2Fds3P1ad2/l7m0Jn91Edz8NmAT8Idos93nnvB9/iLb3aPmQqFV6O6ADodFVmfx+uPtyYLGZ7RUtOhqYQwX/vAlVBQeZWZ0orpzzrtCfd4LS+HzzO4ZUZnG3aiytG6GF7jeE1sV/jTueEp7DoYSivS+BGdHteEJ96f+Ab4F3gcbR9gY8FJ3zLCAtYV9nExpZzQfOSlieBsyOXvMg0WiWZeUG9OaXXgZ7Ev7BzwdeAmpGy2tFz+dH6/dMeP1fo3ObR0KL+rL6/QD2A6ZFn/krhFbkFf7zBm4C5kaxPU3oKVDhPm/gOUI7ie2EEqFzSuPzze8YulXum4YuFhERkUpTZSAiIiIFUEIgIiIiSghERERECYGIiIighEBERERQQiCCmWWZ2Qwzm2lmn5vZwYVs39DMLizCft8zs7QSxjTezBqW5LUiIiWhhEAENrv7fu7eDbgW+Ech2zckzLCXMu5+vIeJjERESoUSApEd1Qd+hDBnhJn9Lyo1mGVmOTPi3Q78JipVuCPa9i/RNjPN7PaE/Q0ys8/M7BszOyz3wcyshZl9EO1rds42ZrbQzJqa2fnRuhlm9r2ZTYrW9zGzKVFsL0XzW4iIlJgGJpJKz8yyCCO/1QJaAEe5+/ScqXfd/Sczawp8Qhj+dg/CaIldo9cfB/wNOMbdN5lZY3dfY2bvAdPd/QozOx643N2PyXXsK4Ba7n6rmVWNjrc+mrchzd1XRdtVByYS5rGfQhjf/zh332hmfyGM2ndzKt8nEanYqhW+iUiFt9nd9wMws17AKDPrShgq9jYzO5ww7fLu5D1N7DHAk+6+CcDdE+e3z5mAajrQNo/XTgVGRhf8V9x9Rj4x3kcYo/+1aPbHLsDHYYh6ahCSBBGRElNCIJLA3adEpQHNCOPdNwN6uPv26Fd7rWLucmt0n0Uef2/u/kGUcPQDnjKzu919VOI2ZnYmoVTi4pxFwDvufkoxYxERyZfaEIgkMLNOQFXC9LkNgJVRMnAk4aIMsB6ol/Cyd4CzzKxOtI/GxTjeHsAKdx8BPE6Y3jhxfQ/gSuB0d8+OFn8CHGJm7aNtdjGzjsU7UxGRHamEQARqm1lOUb0Bw9w9y8yeBV4zs1mEGQfnArj7ajP72MxmA2+6+1Vmth8wzcy2AeOB64p47N7AVWa2HdhAmKI20cVAY2BSVD0wzd3PjUoNnjOzmtF2/0eYvU9EpETUqFBERERUZSAiIiJKCERERAQlBCIiIoISAhEREUEJcN5hUAAAABdJREFUgYiIiKCEQERERFBCICIiIsD/AzjSbRzqavU2AAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgQAAAEWCAYAAAAZ9I+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd5xU1fnH8c9DB0GKoIKICwrSFIQNFhAxCqIUK4oFEbuJmqgxUWPUmFgSS4xYsCFqglGxBBU0ioj4A6WoSBEEkbb0vhSBZZ/fH+eujOt2dvZu+b5fr3nN3P7cmdm9z5xz7jnm7oiIiEjFVinuAERERCR+SghERERECYGIiIgoIRARERGUEIiIiAhKCERERAQlBOWOmbmZHZbH8mFm9qeE6WvMbJWZbTGz/YpwvJTomFWKGnNxic6hRQkcp1l0rMolcKwDzOwTM0s3s4eSfTwpODO7zcyejTsOkeKihKCIzGyRme00s4bZ5n8ZXSBToukR0fTp2db7RzT/kmj6EjPbHV1otpjZ92b2vJm1Ks643f1qd/9LdMyqwMNAL3ev7e7r8kso9kZ0jp8W074+NrPLE+dF57CwOPaf7ViLzOzkhOMsiY61u7iPlYMrgbXAvu5+097uzMwam9loM1ue+D1NWF7dzIab2WYzW2lmN2ZbfpKZzTWzbWY23swO2duY4mBmd5nZvwqxfg8zW5Y4z93vdffLc9tGpKxRQrB3vgfOz5owsyOAWjms9y1wccJ6VYBzge+yrTfZ3WsDdYGTge3AdDNrXxzB5vCL9gCgBjC7OPYvSXEIMMeL0INYLqU2mcB7wNm5bHYX0DI67onA782sd7S/hsAbwJ+ABsA04JXCxlVYpaH0SaRCcHc9ivAAFgG3A1MT5j0I/BFwICWaNyKavwqoH83rC4wFPgUuieZdAnyaw3HeAUblEcfNwApgOXBpdOzDEo79JDAG2EpIMkYAfwVaRfMc2AJ8BHwSTW+N5p2Xw/EqR+ezFlgI/Drapkq0vC7wXBRTWnSsykAb4Adgd7TvjdH61aP9LYneo2FAzYTjnQ58BWwmJFC9gXui/fwQ7euxaN3Ec68LvAisARZHn1WlxPc6Ou4GQmJ3ai7v70uEi+j26Fi/B1KynfPH0XlOitZ5G9gP+HcU99Ss70O0fmvgA2A9MA84N5djjwB2ATuj/Z4cvV+PRJ/38uh19Wj9HsAy4A/ASuClPL43VUj4nibMX04oMcqa/gvwn+j1lcCkhGX7RO9L6zz+Rm4F5kTv8/NAjYTlfaPPdmP03h2Zbds/AF8DO6J4FxG+718TvqPPEZLasUA68CF7/sZ6AMtyiOdkwndoZ/TebgFmRMuHAN9E+1oIXJXtPDOj9bcATQjJ078S9t+fkFxvjL4TbbId+3dR7JsIiVSNaFlDwt/5xug7MZHou6qHHiX5iD2AsvpI+Ocyj3Cxqxz9Mz6EnycEfwWeBq6J5r1KKFkoSEJwKbAqlxh6Ey6i7aN/WiP5eUKwCehKKA2qkRVPtDyFhAtbNO/H7XM55tXAXOBgwq/E8fz04vgm8FQUz/7AlIR/rD87R+AfwOhoX3UIF9P7omVdovh7RvEfRHTxif7hXp5tX4nn/iLw32ifKYRSmssS4tgFXBF9btcQLoSW12edMP2T9y2KZQFwKCERmRMd72TChexF4Plo3X2ApYSLTxXgKEJy1TaXY//4eUXTdwOfRe9tI8KF9C/Rsh5ABvA3QuJQM6d9Ruv+LCEA6kfzDkiYdw4wM3r9T+DJbPuZBZydx/s2K+G78n/s+e4dBawGjo4+g8HR+tUTtv0q2rZmwrzPCEnAQdH2X0T7qkFIau9MeC9yTAii13eRcDGP5vWJPkMDTgC2AZ3y2N+P+2BPgt0TqEpIHBcA1RKOPYWQSDQgJB5XR8vuIyTCVaPH8eTyXdRDj2Q+VGWw914iVAf0JPyRp+Wy3ovAxWZWj/DP5q0C7n854R9ITs4lXGhmuftWwj+o7P7r7v/n7pnu/kMBj5mXc4FH3H2pu68n/DMDQgM44DTgt+6+1d1XEy74A3PakZkZ4VfnDe6+3t3TgXsT1r8MGO7uH0Txp7n73PwCjKpGBgK3unu6uy8CHgIGJay22N2f8dAO4AWgMeFCU1TPu/t37r6J8Iv1O3f/0N0zgNcIFy0Iv4oXufvz7p7h7l8CrwMDCnicC4G73X21u68B/pztvDIJF8Ud7r69kOdQO3relDBvEyGpylq+iZ9KXJ6TxxK+K/ewp4rtSuApd//c3Xe7+wuEkoBjErZ9NNo28TyGuvsqd08j/JL+3N2/jL7bb7LnfS40d383+gzd3ScA/yNcnAviPODd6Lu6i1D6VBM4Ltv5LI/ei7eBjtH8XYTv3yHuvsvdJ7q7BpmREqe6ub33EqGovTnhop8jd//UzBoRqhTecfft4XqYr4MIxYg5aQJMT5henMM6SwtykEJokm2ficc8hPALZ0XCuVXKI4ZGhDYX0xPWN8IvRgi/DscUIcaGURyJsS0mvJdZVma9cPdt0fFrU3SrEl5vz2E6a9+HAEeb2caE5VUI36OCaMLPz6tJwvSavUj8tkTP+xKqY7Jepycs3zfbNonLc5L9u5IV6yHAYDO7LmF5NX56Ljl9bwr6PheamZ0K3En4tV+J8N2cWcDNf/K5uHummS0ll+8cofQh61wfICTz/4u+h0+7+/1FOAWRvaISgr3k7osJddCnERpc5eVfwE3kkTjk4EzCL6GcrCBcNLM0yynEQhyrIPI65lLCr7yG7l4veuzr7u1yiWUt4Z94u4T163poWJm1v0NziSOv81pL+NWV2AK+GbmX3uSnON/DpcCEhPOt5+GOhWsKuP1yfn5eyxOmixyru28gfL4dEmZ3YE+j09mJy8xsH8Lnk1ej1OzflaxYlwL3ZHsfarn7y4khFe1MgFB8/2MD36jUqFFu+zaz6oSSmgcJVSb1CMmo5bR+Dn7yuUSlXwdTgO9cVIp1k7u3ILRDuNHMTspvO5HipoSgeFwG/DIqts/Lo4SqhU/yWsnMKptZczMbSqi7/HMuq74KXGJmbc2sFuHXzd5aBeR1L/+rwPVm1tTM6gO3ZC1w9xWEYtaHzGxfM6tkZoea2QkJ+25qZtWi9TOBZ4B/mNn+AGZ2kJmdEq3/HDAkutWtUrSsdX5xRtUArwL3mFmd6Na4GwkJWVHk954UxjtAKzMbZGZVo8cvzKxNAbd/GbjdzBpFrf7voJDnZWY1CG0MAKpH01lejPZfP3qvryC0Y4BQJN/ezM6OtrkD+DqfapxfR9+VBoTSsay7Ep4Brjazoy3Yx8z6mFle1Q+F8S1QI9pnVUKj0uoJy1cBKWaW9T+wWrR8DZARlRb0yrb+fmZWN5fjvQr0ib6rVQmJ/w5CG488mVlfMzssSiI2ERrMZhb0REWKixKCYhDVO04rwHrr3X1cHvWDx5rZFkLL9I8JxbG/cPcciy3dfSyhlflHhAZMHxUl/mzuAl4ws41mdm4Oy58B3gdmEBp0ZS8VuZjwzzWrZfkoQv0oUXyzgZVmtjaa94co9s/MbDOhpfjh0flNITS++wfhH+UE9vwK+ydwjpltMLNHc4jzOsKvxIWExpsjgeEFewt+5j7CRXKjmf2uiPsAwq9BwoVmIOFX5Ur2NAIsiL8Sbvf7mlCc/UU0rzCy7piA0EA0sY7+TsLdHIsJ7/cD7v5eFPsawu2K9xA+26PJpX1IgpGEJHFhtN+/RvuaRkg2Hov2tYDQ2LNYRG05fgU8S/iVvpXQ6DfLa9HzOjP7Ivpcridc2DcAFxAau2btby4hGVsYfQ8SqzZw93nARcBQQglVP6Cfu+8sQLgtCd/7LcBk4Al3H1+4MxbZe6a2KyKSDGa2iHAnyIdxxyIi+VMJgYiIiCghEBEREVUZiIiICCohEBEREcpZx0QNGzb0lJSUuMMQESkzpk+fvtbdG+W/ppR35SohSElJYdq0fO/+ExGRiJnl1MOpVECqMhARERElBCIiIqKEQERERFBCICIiIighEBEREZQQiIiICEoIREREBCUEIiIighKCEvH993DssTBhQtyRiIiI5EwJQQmoUwcuvBAOOijuSERERHJWrrouLq0aNoRrr407ChERkdwlrYTAzIab2Wozm5XL8pvN7KvoMcvMdptZg2jZIjObGS0r84MTLF4Ms2eDRpoWEZHSKplVBiOA3rktdPcH3L2ju3cEbgUmuPv6hFVOjJanJjHGEvGPf8Axx4BZ3JGIiIjkLGkJgbt/AqzPd8XgfODlZMUSt7Q0tR8QEZHSLfZGhWZWi1CS8HrCbAf+Z2bTzezKfLa/0symmdm0NWvWJDPUIlu+XAmBiIiUbrEnBEA/4P+yVRd0c/dOwKnAr82se24bu/vT7p7q7qmNGjVKdqxFohICEREp7UpDQjCQbNUF7p4WPa8G3gS6xBBXscjMVAmBiIiUfrHedmhmdYETgIsS5u0DVHL39Oh1L+DumELca+7w6qtw6KFxRyIiIpK7pCUEZvYy0ANoaGbLgDuBqgDuPixa7Uzgf+6+NWHTA4A3LTTJrwKMdPf3khVnslWuDGecEXcUIiIieUtaQuDu5xdgnRGE2xMT5y0EOiQnqpK3dCnMmQPHHw+1asUdjYiISM5KQxuCcm3sWOjdG9atizsSERGR3CkhSLK0tNAh0YEHxh2JiIhI7pQQJFlaGhxwAFStGnckIiIiuVNCkGTLl0OTJnFHISIikjclBEmmTolEJJlGjICLLgp9nojsDQ1/nGQvvqhBjUSk+GVmwu23w333wUknwfbtsM8+cUclZZkSgiTrUG5uoBSR0mLbNhg8GEaNgiuugMcfVzsl2XuqMkiidevg2Wdh2bK4IxGR8mLlSujRA15/HR58EJ56SsmAFA8lBEn0zTche589O+5IRKQ8mDkTjj46/E954w246SZVSUrxUUKQRGlp4VmNCkVkb40dC127QkYGTJyoLtGl+CkhSKLly8OzEgIR2RtDh0LfvnDYYTBlCnTqFHdEUh4pIUiitDSoWRPq1Ys7EhEpizIy4Lrr4PrrQ0LwySf6gSHJo4QgibL6IFAdn4gU1ubN0L8/PPYY3HhjaDNQu3bcUUl5ptsOk+ixx2D9+rijEJGyZsmSUCIwZw4MGwZXXRV3RFIRKCFIov32Cw8RkYKaMiWUDPzwQ2hI2LNn3BFJRaEqgyRxh7/8BT77LO5IRKSsGDUKTjgBatWCSZOUDEjJUkKQJBs2wB13wOTJcUciIqWde+iCeMAAOOqo8EOibdu4o5KKRlUGSaI+CESkIHbuhCuvhBdegPPPh+HDoUaNuKOSikglBEmihEBE8rNuHfTqFZKBO++Ef/9byYDEJ2kJgZkNN7PVZjYrl+U9zGyTmX0VPe5IWNbbzOaZ2QIzuyVZMSZTVqdETZrEG4eIlE7z58Oxx4ZqxX/9C+66S7coS7ySWWUwAngMeDGPdSa6e9/EGWZWGXgc6AksA6aa2Wh3n5OsQJNBCYGI5GbCBDjrLKhUCcaNg27d4o5IJIklBO7+CVCUu/C7AAvcfaG77wT+A5xerMGVgNtug9WroXr1uCMRkdJkxIhw98D++8PnnysZkNIj7jYEx5rZDDMba2btonkHAUsT1lkWzcuRmV1pZtPMbNqaNWuSGWuhVKoEjRrFHYWIlBaZmeGHwpAh0L17qCpo0SLuqET2iDMh+AI4xN07AEOBt4qyE3d/2t1T3T21USm6At99d2ggJCKybRucd164tfCKK0KHQxrjREqb2BICd9/s7lui12OAqmbWEEgDDk5YtWk0r0x54gn4+OO4oxCRuK1cCT16wOuvw4MPwlNPQdWqcUcl8nOx9UNgZgcCq9zdzawLITlZB2wEWppZc0IiMBC4IK44i2LXrtB+QLccilRsM2eGMQnWrg2DE51xRtwRieQuaQmBmb0M9AAamtky4E6gKoC7DwPOAa4xswxgOzDQ3R3IMLNrgfeBysBwd5+drDiTYeXK0POYEgKRimvs2FBNUKcOTJwInTrFHZFI3pKWELj7+fksf4xwW2JOy8YAY5IRV0lQp0QiFdtjj8FvfgNHHglvvw1Nm8YdkUj+4r7LoFzasAGqVVNCIFLRZGTAddeFR9++oWRAyYCUFRrLIAlOPTUMXeoedyQiUlI2b4aBA0NVwY03wt//DpUrxx2VSMEpIUgSM3VDKlJRLFkSSgTmzIFhw+Cqq+KOSKTwlBAkwQMPhEFL7r8/7khEJNmmTIH+/WH79lA60LNn3BGJFI0SgiQYOxZ27Ig7ChFJtlGjYNAgOPBA+OgjaNs27ohEik6NCpMgLU0NCkXKM/fQ6+CAAXDUUWFMAiUDUtYpIShm7koIRMqznTvh0kvDuATnnx9KBvbfP+6oRPaeEoJilp4OW7dq2GOR8mjdOujVK4xYeOedYbySGjXijkqkeKgNQTHbsAEOPRRSUuKORESK0/z50KcPLF4M//oXXHhh3BGJFC8lBMXskENgwYK4oxCR4jRhApx1VhjWfNw46NYt7ohEip+qDERE8jBiRLiVcP/94bPPlAxI+aWEoJg991z455GREXckIrI3MjPhj3+EIUOge3eYNClUB4qUV6oyKGZffgnTpkEVvbMiZdb27XDxxaGfgSuugMcfh6pV445KJLl02SpmuuVQpGxbuRJOPx2mToUHHwzjEqgbcqkIlBAUs+XLlRCIlFUzZ4YxCdauhTfegDPOiDsikZKjNgTFTCUEImXT2LHQtWto/zNxopIBqXiUEBQjd2jdGjp0iDsSESmMxx4LJQOHHhq6Ie7UKe6IREqeqgyKkRl8+GHcUYhIQWVkhDYCQ4dCv34wciTUrh13VCLxSFoJgZkNN7PVZjYrl+UXmtnXZjbTzCaZWYeEZYui+V+Z2bRkxSgiFVd6emg8OHQo3HADvPmmkgGp2JJZZTAC6J3H8u+BE9z9COAvwNPZlp/o7h3dPTVJ8RW7sWOhTRv49tu4IxGRvCxZEtoLvP8+DBsGDz8MlSvHHZVIvJJWZeDun5hZSh7LJyVMfgY0TVYsJeX772HuXKhTJ+5IRCQ3U6ZA//6hr4GxY0NHYiJSehoVXgaMTZh24H9mNt3MrsxrQzO70symmdm0NWvWJDXI/KSlhV8ZGgpVpHQaNQpOOAFq1oTJk5UMiCSKPSEwsxMJCcEfEmZ3c/dOwKnAr82se27bu/vT7p7q7qmNGjVKcrR5S0uDxo1V9ChS2rjDfffBgAFw1FHhToK2beOOSqR0iTUhMLMjgWeB0919XdZ8d0+LnlcDbwJd4omwcJYvhyZN4o5CRBLt3AmXXgq33QYDB8JHH6kUTyQnsd12aGbNgDeAQe7+bcL8fYBK7p4eve4F3B1TmIWSmqr+zkVKk/Xrw7DFEybAHXfAXXepG2KR3CQtITCzl4EeQEMzWwbcCVQFcPdhwB3AfsATFv5CM6I7Cg4A3ozmVQFGuvt7yYqzON17b9wRiEiW+fOhTx9YvBj+9S+48MK4IxIp3ZJ5l8H5+Sy/HLg8h/kLgTLX1597eNavD5H4TZgQSgYqVYJx46Bbt7gjEin9Ym9UWF4sWBBuN3zzzbgjEanYRowIdw/svz989pmSAZGCUkJQTNLSYOtW2HffuCMRqZgyM+GPf4QhQ+D442HSpDA2gYgUjMYyKCZpaeFZIx2KlLzt2+Hii0M/A5dfDk88oQa+IoWlhKCYLF8enpUQiJSslSvDmARTp8IDD8BNN6ktj0hRKCEoJmlpoQ2Bui0WKTkzZ4Zhi9euhTfegDPOiDsikbJLCUEx6dIFqlePOwqRimPsWDjvvJCET5wInTrFHZFI2aZGhcXkggvgb3+LOwqRiuGxx0LJwKGHhm6IlQyI7D0lBMVk8+Y9fRGISHJkZMD118N114VOhyZOhKZlfpxUkdJBCUExyMyEhg3h9tvjjkSk/EpPD40Hhw6FG24IfX7Urh13VCLlh9oQFIO1a2HXLjjwwLgjESmfliwJVQRz5sCTT8LVV8cdkUj5o4SgGKgPApHkmTIF+vcPfQ2MGQO9esUdkUj5pCqDYqA+CESSY9QoOOEEqFkTJk9WMiCSTEoIikFWCUGTJvHGIVJeuMN998GAAXDUUeFOgrZt445KpHxTlUEx6NgRbr1VbQhEisPOnXDVVWGQooED4fnnoUaNuKMSKf/yTQjM7FjgIuB4oDGwHZgFvAv8y903JTXCMqBLl/AQkb2zfn0YtnjCBLjjDrjrLnVDLFJS8kwIzGwssBz4L3APsBqoAbQCTgT+a2YPu/voZAdami1eDHXrQr16cUciUnbNnx/6Fli8GF56CS66KO6IRCqW/EoIBrn72mzztgBfRI+HzKxhUiIrQ/r3h5QU+O9/445EpGyaMCGUDJjBuHHQrVvcEYlUPHk2KsxKBsxsHzOrFL1uZWb9zaxq4joVWVqaGhSKFNULL0DPnrD//qHxoJIBkXgU9C6DT4AaZnYQ8D9gEDAiWUGVJT/8AOvW6ZZDkcLKzIQ//hEuuQSOPx4mTQpjE4hIPAqaEJi7bwPOAp5w9wFAu3w3MhtuZqvNbFYuy83MHjWzBWb2tZl1Slg22MzmR4/BBYyzxK1YEZ6VEIgU3PbtYaTCe++Fyy+H996D+vXjjkqkYitwQhDdbXAh4e4CgMoF2G4E0DuP5acCLaPHlcCT0cEaAHcCRwNdgDvNrFT+u1AvhSKFs3Il9OgBr78ODzwATz8NVavGHZWIFDQh+A1wK/Cmu882sxbA+Pw2cvdPgPV5rHI68KIHnwH1zKwxcArwgbuvd/cNwAfknVjEpnlzeOIJOPLIuCMRKf1mzoSjjw7Pr78Ov/udbisUKS0K1DFRdGH/JGF6IXB9MRz/IGBpwvSyaF5u83/GzK4klC7QrFmzYgipcA46CK65psQPK1LmvPcenHtuGKFw4kTo3DnuiEQkUZ4lBGb2jJkdkcuyfczsUjO7MDmhFYy7P+3uqe6e2qhRoxI//rx5MHt2iR9WpEx57LHQx8Chh4bBipQMiJQ++ZUQPA78KUoKZgFrCB0TtQT2BYYD/96L46cBBydMN43mpQE9ss3/eC+OkzR33QVTp8KCBXFHIlL6ZGTAjTfC0KHQrx+MHBlKCESk9MkzIXD3r4Bzzaw2kMqerou/cfd5xXD80cC1ZvYfQgPCTe6+wszeB+5NaEjYi9CGodRJS1ODQpGcpKeHsQjGjIEbbggNCCsXpCmyiMSioG0ItlCEX+hm9jLhl35DM1tGuHMgq0OjYcAY4DRgAbANGBItW29mfwGmRru6293zapwYm7S00EhKRPZYsgT69oU5c+DJJ+Hqq+OOSETyk9TRDt39/HyWO/DrXJYNJ1RJlFruKiEQyW7q1FA9sH17KB3o1SvuiESkIAp626HkYMMG2LFD3RaLZHn9dTjhBKhZM/Q8qGRApOwoVEJgZrWSFUhZVLMmvP12GNxIpCJzh/vug3POgY4dw5gE7fLty1RESpMCJQRmdpyZzQHmRtMdzOyJpEZWBtSsGepJ1f+6VGQ7d8Kll8Jtt4VGhB99FAYqEpGypaAlBP8g9B64DsDdZwDdkxVUWTFvHrz7bviHKFIRrV8fqgVGjIA77gi3FdaoEXdUIlIUBa4ycPel2WbtLuZYypxXXw0lBO5xRyJS8ubPh2OOgcmT4aWX4M9/VjfEImVZQe8yWGpmxwFuZlUJYxt8k7ywyoa0NGjYEKpXjzsSkZI1YQKcdVZIAMaNg27d4o5IRPZWQUsIribcHngQoRfBjuRyu2BFolsOpSJ64QXo2RMaNQqNB5UMiJQPBe2YaC1h6GNJsHy5EgKpODIz4U9/gnvvhV/+EkaNgvqlclByESmKAiUEZtYcuA5ISdzG3Sv0DXdpaRqkRSqG7dvh4otDEnD55WHI76pV445KRIpTQdsQvAU8B7wNZCYvnLLl/fehlnpmkHJu5Uo4/fTQA+EDD8BNN6nxoEh5VNCE4Ad3fzSpkZRBHTrEHYFIcs2cGe6kWbMm9EJ45plxRyQiyVLQRoX/NLM7zexYM+uU9UhqZKXckiUwbBisWhV3JCLJ8d570LUr7NoFEycqGRAp7wpaQnAEMAj4JXuqDDyarpCmToVrrgn3YR9wQNzRiBSvxx+H66+HI46Ad96Bpk3jjkhEkq2gCcEAoIW7q0++SFpaeNZdBlKeZGTAjTfC0KGhquDll6F27bijEpGSUNAqg1lAvWQGUtakpUG1aqFjIpHyID09NB4cOhRuuAHeekvJgEhFUtASgnrAXDObCuzImlmRbztMSwvDHqu1tZQHS5aEEoE5c+DJJ+Hqq+OOSERKWkETgjuTGkUZtHx5SAhEyrqpU6Ffv9DXwJgxYbAiEal4CtpT4YRkB1LWjBoFW7bEHYXI3nn9dRg0KDSMHTcO2rWLOyIRiUuebQjM7NPoOd3MNic80s1sc347N7PeZjbPzBaY2S05LP+HmX0VPb41s40Jy3YnLBtdlJNLpgYNoFmzuKMQKRp3uP9+OOec0J/G558rGRCp6PIrIdgHwN3rFHbHZlYZeBzoCSwDpprZaHefk7WOu9+QsP51wFEJu9ju7h0Le9ySsGXLnn+mHUtlhCK527kztBF4/nkYOBCGD4eaNeOOSkTilt9dBr4X++4CLHD3hdHtiv8BTs9j/fOBl/fieCVm6VK45x74psIPAC1lzfr1cMopIRm44w4YOVLJgIgE+ZUQ7G9mN+a20N0fzmPbg4ClCdPLgKNzWtHMDgGaAx8lzK5hZtOADOB+d38rl22vBK4EaFZCZfhZfRCoUaGUJfPnQ58+sHgxvPQSXHRR3BGJSGmSX0JQGagNJPvmuoHAKHffnTDvEHdPM7MWwEdmNtPdv8u+obs/DTwNkJqaujclGgWmTomkrJkwAc46K9wmO24cdOsWd0QiUtrklxCscPe7i7jvNODghOmm0bycDAR+nTjD3dOi54Vm9jGhfcHPEoI4LF8enpUQSFnwwgtwxRXQogW8+y4cemjcEYlIaZRfG4K9KRmYCrQ0s+ZmVo1w0f/Z3QJm1hqoD0xOmFffzKpHrxsCXYE52beNy8qVUL++6l6ldMvMhNtvh0sugeOPh8mTlQyISO7ySwhOKuqO3T0DuBZ4H/gGeNXdZ4VF6NcAACAASURBVJvZ3WaW2MPhQOA/7p5Y3N8GmGZmM4DxhDYEpSYheOQRWLQo7ihEcrd9e7iD4J574LLLwsiF9evHHZWIlGb20+tw2ZaamurTpk2LOwyRWK1cGcYkmDoV/vY3+N3v1MW25M7Mprt7atxxSPwKOriRJPjtb+GNN+KOQuTnZs6Eo48Oz6+/DjffrGRARApGCUEh7d4Njz0GX3wRdyQiP/Xee9C1K+zaBRMnwplnxh2RiJQlSggKadWqkBToDgMpTR5/PPQx0KIFTJkCnTvHHZGIlDVKCApJfRBIabJ7N/zmN3DttXDaafDpp9C0adxRiUhZpISgkLL6IFAvhRK39PTQePDRR0O7lrfegtq1445KRMqqAg1/LHukp4f+B1RCIHFasgT69YPZs+HJJ8NgRSIie0MlBIV00UWwdSsceGDckUhFNXUqdOkS+sIYM0bJgIgUDyUERWCmW7kkHq+/DiecEEqpJk2CXr3ijkhEygslBIX0hz/AX/4SdxRS0bjD/ffDOedAhw7w+efQrl3cUYlIeaI2BIX0zjvQunXcUUhFsnNnqBZ4/nk477zwrHE0RKS4qYSgkNLSdIeBlJz16+GUU0IS8Kc/wciRSgZEJDlUQlAIW7fCpk26w0BKxvz50LdvaDz44oswaFDcEYlIeaaEoBCy+iBQQiDJNmECnHVWaLw6bhx06xZ3RCJS3qnKoBC2bYNWreCQQ+KORMqzF16Anj2hUaPQeFDJgIiUBJUQFEKHDjBvXtxRSHmVmQl33AH33AO//CWMGgX168cdlYhUFEoIREqB7dth8GB47TW47LLQ+2DVqnFHJSIViaoMCuH++6F//7ijkPJm1So48cRQIvD3v8MzzygZEJGSpxKCQpg+Hb79Nu4opDyZOTPcSbBmTeiF8Mwz445IRCoqlRAUQlqa7jCQ4vPee9C1K+zaBZ98omRAROKV1ITAzHqb2TwzW2Bmt+Sw/BIzW2NmX0WPyxOWDTaz+dFjcDLjLCglBFJcHn8c+vSBFi1gyhRITY07IhGp6JJWZWBmlYHHgZ7AMmCqmY129znZVn3F3a/Ntm0D4E4gFXBgerTthmTFm5/MTFixQr0Uyt7ZvRtuvBEefTRUFbz8MtSuHXdUIiLJLSHoAixw94XuvhP4D3B6Abc9BfjA3ddHScAHQO8kxVkg27ZB9+5w5JFxRiFlWXo6nH56SAZ++1t46y0lAyJSeiSzUeFBwNKE6WXA0Tmsd7aZdQe+BW5w96W5bJtjYb2ZXQlcCdCsWbNiCDtntWvDhx8mbfdSzi1ZAv36wezZ8MQTcM01cUckIvJTcTcqfBtIcfcjCaUALxR2B+7+tLununtqo0aNij1Akb01dSocfXQYk+Ddd5UMiEjplMyEIA04OGG6aTTvR+6+zt13RJPPAp0Lum1JGzECWraEtWvjjELKmtdfhxNOgBo1YNKkMHKhiEhplMyEYCrQ0syam1k1YCAwOnEFM2ucMNkf+CZ6/T7Qy8zqm1l9oFc0LzYLF8J330G9enFGIWWFe+jI6pxzQpfXn38O7drFHZWISO6S1obA3TPM7FrChbwyMNzdZ5vZ3cA0dx8NXG9m/YEMYD1wSbTtejP7CyGpALjb3dcnK9aCSEuDAw6AKurKSfIxcyb89a/w6qtw3nnw/PNQs2bcUYmI5M3cPe4Yik1qaqpPmzYtKfvu1CkMNDNuXFJ2L2VcRgaMHg1Dh8LHH4cqgltvhdtvh0pxt9QRyYOZTXd39YQhsTcqLBO2bIEZM+C44+KOREqbdevgb3+DQw+Fs88OVUt/+xssWxZGLlQyICJlhQrAC2DrVrj0UujVK+5IpLSYMSOUBvz73/DDD2FwokceCbcWqlpJRMoi/esqgAMOCCPQScWWkRE6Exo6NIw9ULMmXHwxXHstHHFE3NGJiOwdJQQFsGxZ6LJYxb8V09q1ISF88klYuhRSUuCBB0KpUYMGcUcnUj5Mnz59/ypVqjwLtEfV2cmQCczKyMi4vHPnzqtzWkEJQT7coWNHGDAgXBCk4vjyy1AaMHIk7NgBJ50Upvv2hcqV445OpHypUqXKswceeGCbRo0abahUqVL5ae1eSmRmZtqaNWvarly58lnCbf4/o4QgH/Pnh4ZjnTvnv66Ufbt2hWqBRx+FTz+FWrXgkktCtUD79nFHJ1KutVcykDyVKlXyRo0abVq5cmWu/8mUEORj0qTwfOyx8cYhybVmTagWeOKJ0OdE8+bw0EMwZEi43VREkq6SkoHkit7fXKtjlBDkY/JkqFsX2rSJOxJJhi++CNUAL78cqgVOPjlUDZ12mqoFRKRiUcONfEyaBMccowaF5cmuXfDKK9CtW6gKeu210EBw9mz44INw66CSAZGKZd68edVatmxZajsY79+/f/OUlJT2LVu2bDdgwICUHTt2WHEfQ5e5fNx7L9x8c9xRSHFYvTp0KZySAgMHwooV8PDD4S6SJ56Atm3jjlBEJGcXXnjh+oULF86aN2/e7B9++MEeeeSRhsV9DCUE+ejXL7Qul7Jr+nQYPBgOPhj+9KfQOPCdd0KD0Rtu0IBVIvJTc+bMqdamTZu2EyZMqJWRkcFVV13VtH379m1atWrV9oEHHmgIkJmZyVVXXdW0ZcuW7Vq1atX2mWeeqQ/wzjvv1ElNTT28R48eh6WkpLS/4IILmu3evZuMjAzOPvvslKz1//znP+9fmJjOO++8TZUqVaJSpUqkpqZuXbZsWbXiPm+1IcjD//1fuO2wW7e4I5HC2rUrDD386KOhHcg++8AVV4S7BVq3jjs6EcnLpZdy8KxZ1CrOfbZvz7bhw1ma33ozZsyoPnDgwEOHDx/+/bHHHrv9wQcfbFi3bt3ds2bN+mb79u32i1/8onW/fv02f/bZZ7VmzpxZ85tvvpm9YsWKKl26dGnTq1evLQAzZ87c58svv5zVqlWrnd27d2/54osv1j/ssMN2rFixour8+fNnA6xdu7ZIFZM7duywV155Zb+HH34433MpLCUEebj7bli5MnRTK2XDqlXw1FMwbFioEjj00NCl8CWXhMahIiK5Wb9+fZUzzjjjsFGjRn3XuXPnHwA+/PDDfefOnVtr9OjR9QHS09Mrz5kzp8bEiRPrnHvuueurVKnCwQcfnHH00Udv+fTTT2vVrVs384gjjtjatm3bnQDnnnvu+okTJ9bu27fv5qVLl1YfPHjwwf369dt05plnbi5KjIMHD252zDHHbOndu/eW4jvzQAlBLjIz4bPP4Pzz445ECmLq1FAa8OqrsHMnnHIKPPss9O6tBqEiZU1BfsknQ506dXY3adJk5/jx42tnJQTubg899NCSs88++ycX8HfffTfXnxhm9rPpRo0a7Z41a9acN998c99hw4Y1euWVVxq89tpri7LWycjIoH379m0BevfuvfGRRx5Znn2/N910U+O1a9dWef/997/buzPNmf5V5mLOHNi8Wf0PlGY7d4ZeBI85Brp0CR0KXXUVzJ0L770Xbh1UMiAiBVW1alUfO3bsdy+//PJ+w4YNawDQs2fPTU8++WSjrFb9X3/9dfXNmzdX6t69e/qoUaMaZGRksHz58ipTpkypffzxx2+FUGUwd+7cart372bUqFENjj/++PQVK1ZU2b17N5dccsnG++67L23mzJk/qRKpUqUKc+fOnTN37tw5OSUDDz/8cMOPPvqo7ltvvbWwcpJug1IJQS4mTw7PGvK49Fm5ck+1wMqV0LJlKB0YPBj23Tfu6ESkLNt3330z33///QU9evRoVadOnd033HDD2kWLFlU/4ogj2ri7NWjQYNeYMWO+GzRo0MZJkybVbtOmTTsz8z//+c/LmjVrlvH111/Tvn37rVdffXWzRYsW1TjuuOM2Dxo0aOOUKVNqXnbZZSmZmZkGcPfddy8rTFy///3vD2ncuPGO1NTUNgB9+/bd8OCDD64oznNXQpCLyZOhYUM47LC4I5Esn38eOhF69dXQaPDUU+H668Ow1CoJEJG9cfjhh+/MavDXsGHD3bNmzfoma9ljjz2WBqRl3+app55aBvzswl6nTp3d48ePX5A479hjj90+Z86cb7KvW1AZGRnTi7ptQSkhyMUTT8Dvfw9W7F0/SGHs2BE6Dho6FKZMgTp14Jpr4Ne/hlat4o5ORKT8SOrvKjPrbWbzzGyBmd2Sw/IbzWyOmX1tZuPM7JCEZbvN7KvoMTqZceakRg3dnhanFSvgzjvhkENg0CDYtCkkBWlp8M9/KhkQkdKpb9++6dlLB8qKpJUQmFll4HGgJ6FIZaqZjXb3OQmrfQmkuvs2M7sG+DtwXrRsu7t3TFZ8eZk8Gd58E/7wB9hvvzgiqJjcQ7XAo4+GUoHdu0PDwOuug549VS0gIpJMyfwX2wVY4O4L3X0n8B/g9MQV3H28u2+LJj8DmiYxngJ7993QpW2NGnFHUjHs2AEvvRTuFDj22PD+X3stfPtt6FHwlFOUDIiIJFsy2xAcBD+5l3QZcHQe618GjE2YrmFm04AM4H53fyunjczsSuBKgGbNmu1VwFkmT4YOHULvdpI8y5eHOwWeeiqMM9C6NTz+OFx8MdSuHXd0IiIVS6loVGhmFwGpwAkJsw9x9zQzawF8ZGYz3f1nnTG4+9PA0wCpqal7PZZ2RkYoth4yZG/3JDlxDwnX0KEwalSoFujbN1QLnHyyGnGKiMQlmQWxacDBCdNNyeG2DTM7Gfgj0N/dd2TNd/e06Hkh8DFwVBJj/dGsWbB1q/ofKG4//AAvvAC/+AV07Qpjx4ZbBufPh9GjQxsBJQMiUpbccsstBxbXvm688cYmd9xxxwFF3X7SpEk1O3bs2Pqwww77yWBLhZHMhGAq0NLMmptZNWAg8JO7BczsKOApQjKwOmF+fTOrHr1uCHQFEhsjJk1aGjRooB4Ki0taGtx+OzRrFsYT2LYt3NK5bBk89FAYa0BEpCx69NFHGxdm/czMTHbv3p2UWGrXrp350ksvfb9gwYLZ//vf/+bfdtttBxd2AKWkJQTungFcC7wPfAO86u6zzexuM+sfrfYAUBt4LdvthW2AaWY2AxhPaENQIglBnz6wdm243U2Kxj2MFHneeeF9vPfekGB9+CHMnh36EVAbAREpTebNm1etefPm7fr379+8RYsW7Xr37t0iPT290ujRo+ucfPLJP/50efPNN/ft2bPnob/61a8O2rFjR6XWrVu37d+/f3OAu+6664CWLVu2a9myZbu77757/6z9pqSktD/zzDNTWrVq1e67776rNmrUqH3btm3b5vDDD2977LHH/ngT9TfffFOzS5cuhzdt2vSIv/71r4UaHvnII4/cccQRR+wASElJ2dWgQYOMFStWFKpZQFLbELj7GGBMtnl3JLw+OZftJgFHJDO2vKjoumh++AFefjm0D/jyS6hXD377W/jVr6BFi7ijE5GypEsXDs8+76yzWH/LLaxJT6fSSSfRMvvyiy5i7fXXs27FCqqcfjo/KX+cMoV5+R1z0aJFNZ566qlFvXr12jpgwICUBx54oNFdd9216je/+U2z5cuXV2nSpEnG8OHD9xsyZMjaCy64YNOIESP2nzt37hyAiRMn1ho5cuR+06dP/8bd6dy5c5uTTjopvWHDhruXLFlS/bnnnvv+pJNOWrR8+fIq1157bcrHH388t3Xr1jtXrVr146/4BQsW1Jg0adK8jRs3Vm7Tpk37m2++eU316tUL3TZu/PjxtXbt2mVt27bdkf/ae+hmrgSrVkGbNmFgHCm4pUvhttvg4IPh0kvDoEPDhoVqgQcfVDIgImXDgQceuLNXr15bAQYNGrRu0qRJtStVqsS555677plnnmmwdu3ayl988UXtAQMGbMq+7ccff1z7tNNO27jvvvtm1q1bN7NPnz4bxo8fXwegcePGO0866aSt0Xr7dOnSJb1169Y7AQ444IAf6xB69eq1sWbNmt64ceOMBg0a7Fq2bFmhf7QvXry46pAhQ1o888wziwo7CFKpuMugtJg8OYyUV6dO3JGUfu7w6aehE6E33wzT/fuHuwVOPFGlLCKyd/L6RV+nDpl5LW/cmIyClAhkl9OwxQDXXHPNuj59+hxWo0YN79ev34aqVasWar+1atXKLMh6iaUBlStXJiMj4ycBvfjii/XuvffeJgBPP/30ou7du29LXL5+/fpKp5566mF33nlnWlYCUhgqIUgweTJUrQqdO8cdSem1fTsMHw5HHQXdu8O4cXDjjfDddyEx+OUvlQyISNm0YsWKah9++OE+AP/+978bHHfccVsg1MkfcMABux566KHGV1555dqs9atUqeJZwyKfeOKJW8aMGVMvPT290ubNmyuNGTOm/oknnpie/Rg9evTYOmXKlDpz586tBpBYZZCfiy++eGPWEMnZk4EffvjB+vTpc9jAgQPXDRkyZENRzl8JQYJJk6BTJ/VQmJMlS+DWW0O1wGWXQWYmPP10qBb4+98hJSXuCEVE9k5KSsoPQ4cO3b9FixbtNm7cWOV3v/vdmqxlAwcOXNe4ceOdnTp1+iFr3oUXXrimTZs2bfv379+8W7du2y644IJ1nTp1atO5c+c2gwYNWtO1a9ft2Y/RpEmTjEcffXTRmWeeedjhhx/e9swzzyyWStXhw4fXnzp1au2RI0c2bN26ddvWrVu3nTRpUs3C7MPc97ovn1IjNTXVp02bVqRtd+6EunVDC/iHHy7mwMood/jkk9BI8M03w7wzzgjVAiecoJIAkfLAzKa7e2rcccyYMWNRhw4d1ua/ZnLMmzevWt++fVtmDYGc3cUXX9zsqKOO2nbDDTfEFmNxmDFjRsMOHTqk5LRMbQgi6enhNrneveOOJH7btsHIkSER+Prr0C/DzTeHZEm3Y4pIRdOuXbs2NWvWzHzqqaeW5r922aWEILLffjBiRNxRxGvx4tBp0LPPwvr1cOSR4fX550OtWnFHJyKSPIcffvjO3EoHZs+e/U1JxxMHJQSRNWugYcOKVwzuDhMmhLsF/vvfcP5nnBG6FT7++Ir3fohIbDIzMzOtUqVK5aceu5TJzMw0INc7HtSoMJKaCpdfHncUJWfbNnjmmVAKcOKJoa3A738PCxeGQYe6d1cyICIlataaNWvqRhctKWaZmZm2Zs2ausCs3NZRCQGhv/0lS8LFsbxbtGhPtcCGDdCxIzz3XKgWqFmo9qgiIsUnIyPj8pUrVz67cuXK9ujHajJkArMyMjJy/emrhIDQ/wCU3xEO3WH8+FAt8Pbb4Zf/WWeFaoGuXVUSICLx69y582qgf74rStIoISD0P1CjBnToEHckxSM9HWbMgOnT4YsvwvktWBDaSNxyC1x9dehPQEREJIsSAkIJwS9+AdWqxR1J4W3YEAYS+uKLPY9vvw2lAgAHHhg6W/rjH2HgQHW6JCIiOVNCQOiBr5BjQMRi9eqfXvi/+AK+/37P8mbNwsX/wgvDc6dO0LhQo3WLiEhFpYSAMChPaeIOy5f//OK/bNmedQ49NJRqXHVVuPAfdVSoEhARESkKJQQxcw8dAmXV92c9Vq8Oy82gdevQVXDWr/6OHaFevXjjFhGR8kUJQQnKzAyN+7L/8t8QjUtVpQq0awd9+uy5+B95JNSuHW/cIiJS/ikhKGYZGaFo//vvwz3/ixaF1wsXhpb/W7aE9apVCxf7AQP2XPyPOEKN/kREJB5KCApp9+5Qv591oU98XrQIli4N62SpVAmaNg3DA19yyZ6Lf9u2ULVqHGcgIiLyc0lNCMysN/BPoDLwrLvfn215deBFoDOwDjjP3RdFy24FLgN2A9e7+/vJjBXCEMjLl4dHWtrPn5cuDT0a7tr10+2aNAkX/K5dw3Pz5nuemzYtm7cziohIxZK0hMDMKgOPAz2BZcBUMxvt7nMSVrsM2ODuh5nZQOBvwHlm1hYYCLQDmgAfmlkrd99NMcvMDK31ly4NAxxlV61auOAfdFAY72DAgJ9e9Js1UzG/iIiUfcksIegCLHD3hQBm9h/gdCAxITgduCt6PQp4zMwsmv8fd98BfG9mC6L9TS7uICtVCsX3v/jFngt/4vN++6lrXxERKf+SmRAcBCxNmF4GHJ3bOu6eYWabgP2i+Z9l2/agnA5iZlcCVwI0a9asSIG+9FKRNhMRESk3yvyIUu7+tLununtqo0aN4g5HRESkTEpmQpAGJA6h0zSal+M6ZlYFqEtoXFiQbUVERKSYJDMhmAq0NLPmZlaN0EhwdLZ1RgODo9fnAB+5u0fzB5pZdTNrDrQEpiQxVhERkQotaW0IojYB1wLvE247HO7us83sbmCau48GngNeihoNrickDUTrvUpogJgB/DoZdxiIiIhIYJ41Tm45kJqa6tOmTYs7DBGRMsPMprt7atxxSPzKfKNCERER2XtKCEREREQJgYiIiJSzNgRmtgZYnM9qDYG1JRBOaaPzrlh03hXL3pz3Ie6uTlykfCUEBWFm0ypiAxqdd8Wi865YKup5S/FSlYGIiIgoIRAREZGKmRA8HXcAMdF5Vyw674qlop63FKMK14ZAREREfq4ilhCIiIhINkoIREREpOIkBGbW28zmmdkCM7sl7niKwswONrPxZjbHzGab2W+i+Q3M7AMzmx8914/mm5k9Gp3z12bWKWFfg6P155vZ4IT5nc1sZrTNo2ZmJX+mOTOzymb2pZm9E003N7PPo1hfiUbVJBol85Vo/udmlpKwj1uj+fPM7JSE+aXy+2Fm9cxslJnNNbNvzOzYivB5m9kN0Xd8lpm9bGY1yuPnbWbDzWy1mc1KmJf0zze3Y0gF5+7l/kEYbfE7oAVQDZgBtI07riKcR2OgU/S6DvAt0Bb4O3BLNP8W4G/R69OAsYABxwCfR/MbAAuj5/rR6/rRsinRuhZte2rc551w/jcCI4F3oulXgYHR62HANdHrXwHDotcDgVei122jz7460Dz6TlQuzd8P4AXg8uh1NaBeef+8gYOA74GaCZ/zJeXx8wa6A52AWQnzkv755nYMPSr2o6KUEHQBFrj7QnffCfwHOD3mmArN3Ve4+xfR63TgG8I/z9MJFw6i5zOi16cDL3rwGVDPzBoDpwAfuPt6d98AfAD0jpbt6+6fubsDLybsK1Zm1hToAzwbTRvwS2BUtEr28856P0YBJ0Xrnw78x913uPv3wALCd6NUfj/MrC7hgvEcgLvvdPeNVIDPmzA0e00zqwLUAlZQDj9vd/+EMPR7opL4fHM7hlRgFSUhOAhYmjC9LJpXZkXFokcBnwMHuPuKaNFK4IDodW7nndf8ZTnMLw0eAX4PZEbT+wEb3T0jmk6M9cfzi5ZvitYv7PsRt+bAGuD5qKrkWTPbh3L+ebt7GvAgsISQCGwCplP+P+8sJfH55nYMqcAqSkJQrphZbeB14LfuvjlxWfRLoFzdS2pmfYHV7j497lhKWBVCcfKT7n4UsJVQvPujcvp51yf8gm0ONAH2AXrHGlRMSuLzLY/fISmaipIQpAEHJ0w3jeaVOWZWlZAM/Nvd34hmr4qKB4meV0fzczvvvOY3zWF+3LoC/c1sEaF495fAPwlFplWidRJj/fH8ouV1gXUU/v2I2zJgmbt/Hk2PIiQI5f3zPhn43t3XuPsu4A3Cd6C8f95ZSuLzze0YUoFVlIRgKtAyaqVcjdDwaHTMMRVaVC/6HPCNuz+csGg0kNWyeDDw34T5F0etk48BNkXFhO8DvcysfvRrrBfwfrRss5kdEx3r4oR9xcbdb3X3pu6eQvjsPnL3C4HxwDnRatnPO+v9OCda36P5A6NW6c2BloRGV6Xy++HuK4GlZnZ4NOskYA7l/PMmVBUcY2a1oriyzrtcf94JSuLzze0YUpHF3aqxpB6EFrrfEloX/zHueIp4Dt0IRXtfA19Fj9MI9aXjgPnAh0CDaH0DHo/OeSaQmrCvSwmNrBYAQxLmpwKzom0eI+rNsrQ8gB7sucugBeEf/ALgNaB6NL9GNL0gWt4iYfs/Ruc2j4QW9aX1+wF0BKZFn/lbhFbk5f7zBv4MzI1ie4lwp0C5+7yBlwntJHYRSoQuK4nPN7dj6FGxH+q6WERERCpMlYGIiIjkQQmBiIiIKCEQERERJQQiIiKCEgIRERFBCYEIZrbbzL4ysxlm9oWZHZfP+vXM7FcF2O/HZpZaxJjGmFm9omwrIlIUSghEYLu7d3T3DsCtwH35rF+PMMJe0rj7aR4GMhIRKRFKCER+al9gA4QxI8xsXFRqMNPMskbEux84NCpVeCBa9w/ROjPM7P6E/Q0wsylm9q2ZHZ/9YGbW2Mw+ifY1K2sdM1tkZg3N7Opo2Vdm9r2ZjY+W9zKzyVFsr0XjW4iIFJk6JpIKz8x2E3p+qwE0Bn7p7tOzht51981m1hD4jND97SGE3hLbR9ufCvwJONndt5lZA3dfb2YfA9Pd/SYzOw240d1Pznbsm4Aa7n6PmVWOjpcejduQ6u5ro/WqAh8RxrGfTOjf/1R332pmfyD02nd3Mt8nESnfquS/iki5t93dOwKY2bHAi2bWntBV7L1m1p0w7PJB5DxM7MnA8+6+DcDdE8e3zxqAajqQksO2U4Hh0QX/LXf/KpcY/0noo//taPTHtsD/hS7qqUZIEkREikwJgUgCd58clQY0IvR33wjo7O67ol/tNQq5yx3R825y+Htz90+ihKMPMMLMHnb3FxPXMbNLCKUS12bNAj5w9/MLGYuISK7UhkAkgZm1BioThs+tC6yOkoETCRdlgHSgTsJmHwBDzKxWtI8GhTjeIcAqd38GeJYwvHHi8s7A74CL3D0zmv0Z0NXMDovW2cfMWhXuTEVEfkolBCJQ08yyiuoNGOzuu83s38DbZjaTMOLgXAB3X2dm/2dms4Cx7n6zmXUEppnZTmAMcFsBj90DuNnMdgFbCEPUJroWaACMj6oHprn75VGpwctmVj1a73bC6H0iIkWiRoUiIiKiKgMRERFRQiAiIiIoIRAREn3ulAAAACBJREFUERGUEIiIiAhKCERERAQlBCIiIoISAhEREQH+H/2kSP0zoJtaAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] @@ -527,22 +473,17 @@ } ], "source": [ - "plot_absolute_time(results, [2], max_batch_size=max_batch_size)" + "plot_absolute_time(experiments_neq, results, [2], max_batch_size=max_batch_size)" ] }, { "cell_type": "code", "execution_count": 11, - "id": "cf6a0dfc", - "metadata": { - "pycharm": { - "name": "#%%\n" - } - }, + "metadata": {}, "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhsAAAEWCAYAAADPUVX+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd5gUVdbH8e8hgySBERFUUAliAkHW+KqorIhZQcwJcWVNGNacs655XRXFgIJZERWzsGaUARWQqGIgg5KVNOf9497RdpzQM0xPTfh9nqee6bqVTldXV5+5deuWuTsiIiIimVIt6QBERESkclOyISIiIhmlZENEREQySsmGiIiIZJSSDREREckoJRsiIiKSURU22TAzN7OtCpn+gJldkTJ+hpnNM7PlZta0BNtrHbdZo6Qxl5b4HrZIOo6yYmaPmdn1xZi/0GOjvDGz183sxKTjSIeZbRaPv+olWLaumb1iZkvM7LlMxCclY2bHmtlbScchlVeRyYaZzTSz1WbWLE/5+HhSbx3HH4vjh+SZ785YflIcP8nM1sUT1nIz+87MHjWzdqX2rgB3/4e7Xxe3WRO4A+jh7vXdfVEmf5Die/ywlNY12sz6pZbF9/Btaay/NJSHRMzMdjGzj5PafrrM7GozezK1zN17uvvjScVUmPj93zd33N1/iMffuhKs7kigOdDU3XuXQmy1zOz5GKOb2V55ppuZ3WJmi+Jwi5lZyvROZpZtZivj307rG1MSinu+ye/76u5D3b1HZiIUSb9m4zvg6NwRM9sOqJfPfNOAE1LmqwH0Ab7JM98n7l4faATsC/wKZJvZtumHXrB8/utqDtQBJpXG+qV0lVKS0gsYWQrrqTISSA43B6a5+9riLlhIrB8CxwFz85nWHzgU2AHYHjgIOD2urxbwMvAksCHwOPByLM+YmABV2BplkRJz90IHYCZwOfB5Stm/gcsAB1rHssdi+Txgw1h2IPA64YRwUiw7Cfgwn+28CjxfSBwXAnOA2cApcdtbpWz7fsKPzQpCAvMYcD3QLpY5sBx4D3g/jq+IZUfls73q8f0sBL4F/hmXqRGnNwIGx5hmxW1VB7YGfgPWxXUvjvPXjuv7Ie6jB4C6Kds7BPgCWEpIzvYHbojr+S2u6z9x3tT33ggYAiwAvo+fVbXUfR23+wshaexZxGd9CfB1nP9RoE6cNhE4KGXemnHfdI7vKXf/Lgd2ISSyl8eY5scYG8VlW8f5T43Lvh/Ldwc+BhYDP/LHMfMYcB/wGrAMGANsmSf2ccCO+eyf3eO69orjHYC3gZ+BqUCflHUUtS8/Av4DLAGmAPukLHsS4ThZFvfzsfns3/2B1cCauJ++jOWjgX55tnNn3A/fArvG8h/jvjwxZZ2FHld5tp+67kWEY3ZLwndiUfw8hwKN4/xPADmEfwaWA/9K+exyvwebACPi/pwBnFbAtq/J895PpQTHSCHH7k+5n3FK2cdA/5TxU4FP4+sehO+tpUz/Adi/gPWPBm4CPiN8R18GmqRM35k/jt0vU2OJy94Q9/2vwFax7Pq4zHLgFaBp3P9Lgc/549z6p32eesxQ8PmmFzA+rutH4Oo87zPv9/UkUs7LhGPuc8Kx/jmwa55tXxffzzLgLaBZnFaHkMAtivvic6B5Ub8zGir/UPQM4QdoX8KJeWvCD+pPhP9S8iYb1wODgDNi2bOEGpF0ko1TgHkFxLA/4US6LbABMIy/JhtLgN0IJ7A6ufHE6fl9WX9fvoBt/oPwg7Ip0AQYxZ9Psi8BD8Z4NiKchE4v6D0STvAj4roaEE4uN8Vp3WL8+8X4WwId4rTRxB+i/GInnKBfjutsTahdOjUljjXAafFzO4OQrFkB73kmIanIfc8fpezDfwHPpMx7CDChkP17CuHHZwugPvAi8ESe+YfE/VeXcDwti8dLTcKJt1PK57so7qcahBPy0ynbakHKD0fu/iEcNz8C3WL5BnH85LiezoQf2I5p7su1wMAY31HxM2sS17sUaJ8SzzYF7OOrgSfzlP3+Gads5+T4mV1P+HG4j5BY9Ij7qX5Rx1U+285d91nx/deN+2m/uO4sQiJ+V97vf8r4nz7rOP9/Cd+5ToRErXs6751iHiNFnKfySzaWAH9LGe8KLIuvBwKv55n/VeD8AtY/mnCM5Z6DXsh9L4Tv6yLgAML3d784npWy7A/ANnG/14xlMwjJXiNCgj+NcK6tEd/3o4V8v/IeM3nPN3sB28V4tiecPw8tZH2/r4NwLP0CHB9jOTqON03Z9jeEf+TqxvGb47TTCcdgPcLx2wVoWNTvjIbKPxSnOu8JwiWS/YDJhC9efoYAJ5hZY2BPYHia659NOMjz04fwxZvo7isIJ628Xnb3j9w9x91/S3ObhelDOOn+6O4/E/6rAcDMmhNOLOe6+wp3n0846ffNb0XxOnF/YKC7/+zuy4AbU+Y/FXjE3d+O8c9y9ylFBRgvF/UFLnH3Ze4+E7idcJLI9b27P+ThGvvjhB/C5oWs9j8p7/kG/rh89iRwgJk1jOPHE46JghwL3OHu37r7ckKNSd881eFXx/33K3AM8I67P+Xua9x9kbt/kTLvS+7+mYcq+KGEH7ZcBwBvuLunlPUmJIM93f2zWHYgMNPdH3X3te4+nvCj0TvNfTmfcEyscfdnCAl4rzgtB9jWzOq6+xx3X59Ldt/FGNcBzxCSv2vdfZW7v0WoIdgqjeMqP7Pd/d74/n919xnxuFvl7gsIbZv2TCdIM9uUkOBf5O6/xc/rYVIupRahuMdIcdUnJBy5lgD1437LOy13eoNC1vdEyjnoCqBPPG6OA0a6+8j4/X0bGEs4LnM95u6T4n5fE8sedfdv3H0JoQb4G3d/Jx7jzxGS4RJx99HuPiHG8xXwFGl+roRjerq7PxHjfYrwj9dBKfM86u7T4ufyLH98H9cQ/lHYyt3XuXu2uy8t6fuQyqM412yfIPwX04aQUOTL3T80syzCZZZX3f3XlDZZhWlJqIrNzyZAdsr49/nM82M6GymGTfKsM3WbmxP+O5mT8t6qFRJDFiHTz05tn0bI/CH8mJSkvUGzGEdqbN8T9mWu369lu/vKuP36hawz73veJC4728w+Ao4ws5eAnsA5haxnk3ziqsGfE53UbW3KX9v2pEq9Jr+SP7+HAwi1XanOBYa4+8SUss2Bv5nZ4pSyGoRjO519OStPQvM9sIm7rzCzo4ALgMFxP52fTsJYgHkpr38FcPe8ZfUp+rjKz5+O0Zg43w3sQfihrUb4LzYdmwC5SU6u7wk1COkuX5xjpLiWAw1TxhsCy93dzSzvtNzpyyhY3u9GTcJxszkhYU39Ma5JqA3Nb9lceT/T/D7jEjGzvwE3E2piahFqrtK9Ayjv5wKFnFf48/fxCcJ3+en4D+eTwGUpCZZUUWnXbLj794Rr0QcQqjsL8yRwPoUkJfk4DPiggGlzCAdwrs3yC7EY20pHYdv8EVhFuE7ZOA4N3X2bAmJZSDh5bJMyfyMPjWRz17dlAXEU9r4WEv6T2DxPnAXVOqUj73uenTL+OOG/uN6ERr6528kvxtn5xLWWP59QU5crbB8UKN5ptCehHUaq3sChZpaaEP0I/C/lM2js4c6KM0hvX7a0P2fOv+8fd3/T3fcj1BxNAR4qIOTSPE6LOq7S2f6NsWw7d29I+HytkPlTzQaamFlqbUBxjr/iHiPFNYnQODTXDvzRSHwSsH2ez3N7Cm9Enve7sYbwGfxIqPVIPa42cPebU+Zfn/exIv5NbZS/cRHrHka4vLapuzcitOWxQuZPlfdzgTQ/11jrd427dyS0+ziQ9Gu6pBIrbqvoUwnXY1cUMd89hMst7xc2k5lVN7M2ZnYv4RrjNQXM+ixwkpl1NLN6wFXFCztf8wjXigvyLHC2mbUysw2Bi3MnuPscQqOo282soZlVM7MtzSy3mnIe0Cq3Zbu75xB+fO40s40AzKylmf09zj8YONnM9onramlmHYqKM1azPwvcYGYNzGxz4DxCsldS/4zvuQmhduqZlGnDgR0JNRqpieQCwmWE1DifAgbGz7c+4UftGS/4ToShwL5m1sfMaphZ0zRvRdwd+CqfqtrZwD7AOWZ2Rix7FWhnZsebWc047GRmW6e5LzciHBM1zaw3oQ3TSDNrbmaHmNkGhCR0edwf+ZkHtC6NOxLSOK7S0YAQ7xIza0loiJ033oKOvx8JDRxvMrM6ZrY94RyR7vFX3GPkL8ystpnViaO1Yhy5P6pDgPPiPtmE8A/QY3HaaEKjyrPjOs6M5e8VsrnjUs5B1xIatK8jvN+DzOzv8ZxWx8z2MrNW6b6PwsTLW7Pi9qub2Sn8OTH/0/kmakCodfrNzLoRLlPmyu/7mmok4XtyTPwuHgV0JHx/CmVme5vZdvHy0lJCQlbQd0GqkGKd8OL1xbFpzPezu7+bp8o51S6xGnMp4UvfENjJ3ScUsL7XgbsIJ4IZFH5CSNfVwONmttjM+uQz/SHgTULL8nH8tTbnBEL1ZO6dG88T/qslxjcJmGtmC2PZRTH2T81sKfAO0D6+v88IDQLvJFw3/h9//GdxN3Ckmf1iZvfkE+dZhP98viU0xB0GPJLeLsjXMEIi9S3hssbvnWnF67MvEC6lvZhSvpLY2j7uz51jDLmX3r4jtJg/q6CNuvsPhFqz8wmX077gz/+VFqTAW17jOvcBLjazfrG6vwehTcNsQlXwLYQqZih6X44B2hL+m70BONLdFxG+R+fFdf5MqGk5g/zlVmUvMrNxaby/ohR4XKXpGkICuYRwt0/e4/wm4PL4uV6Qz/JHExocziY0mr7K3d9Jc9vFOkYKMJVQu9OS8H39lT++Ow8SGitOIDR8fi2W4e6rCbfFnkC4a+IUQgPK1YVs6wlCsjKX0CD27LiuHwkNpi8l/JD/SEjaSvMW19PiOhcRGpqm9imT3/lmAHCtmS0DriQk0sR48/u+kjJ9EaFG4vy4vX8BB7r7Qoq2MeFcuJTQtu9/FN62S6oIKzgfkKrGzGYSWrgX+GNhZlcC7dz9uDILrBBm9jXhR//rDG/nJMK+2T2T25HyycxGE+4+eTjpWEQqosS73paKI15aOZU/36GRmFhtPCTTiYaIiKwf9WQnaTGz0wjVw6+7e6FtccqKu6/O0whPRETKIV1GERERkYxSzYaIiIhkVIVos9GsWTNv3bp10mGIiFQo2dnZC909K+k4RCpEstG6dWvGji3yjlsREUlhZvn1tixS5nQZRURERDJKyYaIiIhklJINERERySglGyIiIpJRSjZEREQko5RsiIiISEYp2RAREZGMUrIhIlIO5eTA44/Ds88WPa9IeadkQ0SknPn0U9hlFzjpJBg6NOloRNafkg0RkXJi1iw44YSQaPz4Y6jZeOmlpKMSWX8VortyEZHK7Lff4I474MYbYc0auOSSMDRokHRkIqVDyYaISELc4cUX4YILYOZMOPxwuO022GKLpCMTKV26jCIikoCvvoLu3eHII6F+fXj3XXjhBSUaUjkp2RARKUMLF8IZZ0DnziHh+O9/Yfz4kHiIVFa6jCIiUgbWrAmJxdVXw7JlcOaZcNVV0KRJ0pGJZJ6SDRGRDHvzTTj3XJgyBfbbD+66Czp2TDoqkbKjyygiIhkybRocdBDsvz+sXQsjRoTEQ4mGVDVKNkREStmSJXDhhbDttvC//8Gtt8LEiSHxMEs6OpGyp8soIiKlZN06eOwxuPRSWLAATj4ZbrgBNt446chEkpXxmg0zq25m483s1Tj+mJl9Z2ZfxKFTpmMQEcm0Dz+Ebt2gXz/Yaiv47DMYPFiJhgiUzWWUc4DJecoudPdOcfiiDGIQEcmIH36Avn1hjz1g/nwYNiwkHl27Jh2ZSPmR0WTDzFoBvYCHM7kdEZGytnJluI21Qwd4+eVwG+uUKXD00WqXIZJXpms27gL+BeTkKb/BzL4yszvNrHaGYxARKTXu8PTTIcm45prQ6HPKlJB4bLBB0tGJlE8ZSzbM7EBgvrtn55l0CdAB2AloAlxUwPL9zWysmY1dsGBBpsIUEUlbdna4XHL00dC0abjT5JlnYPPNk45MpHzLZM3GbsDBZjYTeBrobmZPuvscD1YBjwLd8lvY3Qe5e1d375qVlZXBMEVECjdvXmj4udNOoe+MQYNg7Fj4v/9LOjKRiiFjyYa7X+Lurdy9NdAXeM/djzOzFgBmZsChwMRMxSAisj5Wr4Z//xvatYPHH4eBA0OycdppUL160tGJVBxJ9LMx1MyyAAO+AP6RQAwiIgVyh9deg/POg+nToVcvuP12aN8+6chEKqYySTbcfTQwOr7Wsw1FpNyaPDnUYLz5ZkguRo6Enj2TjkqkYlN35SIiwC+/hIelbbcdfPop3HknTJigREOkNKi7chGp0tatg4cegssvh59/hv794brrQO3SRUqPajZEpMoaNQp23BHOOCM8NG3cOHjgASUaIqVNyYaIVDnffQdHHgndu4cntD73XEg8OulJTSIZocsoIlJlLF8ON98cbmetXj1cLjn/fKhbN+nIRCo3JRsiUunl5IQHpF10EcyeDcceG5KOVq2SjkykatBlFBGp1D77DHbbDY4/HjbZBD76CJ58UomGSFlSsiEildLs2XDiifC3v8HMmfDYYzBmDOy6a9KRiVQ9uowiIpXKb7+FPjJuuAHWrIGLL4ZLL4UGDZKOTKTqUrIhIpWCOwwfHhp8fvcdHHpoaAi65ZZJRyYiuowiIhXehAmw775w+OFQrx68/Ta89JISDZHyQsmGiFRYixbBP/8Z+scYPx7uvRe++CIkHiJSfugyiohUOGvWhJ4+r7oKli6FAQPg6quhadOkIxOR/CjZEJEK5e23wwPTvv4a9tkH7rordDUuIuWXLqOISIUwYwYccgj06BHuOBk+PCQeSjREyj8lGyJSri1dGnr+7NgR3nsv9Pz59dch8TBLOjoRSUfGkw0zq25m483s1TjexszGmNkMM3vGzGplOgYRqXhycuDRR6FdO7j11tDF+LRpIfGoXTvp6ESkOMqiZuMcYHLK+C3Ane6+FfALcGoZxCAiFcjHH0O3bnDKKdCmTehy/NFHoUWLpCMTkZLIaLJhZq2AXsDDcdyA7sDzcZbHgUMzGYOIVBw//RRqMHbbDebMCc8w+fhj2GmnpCMTkfWR6ZqNu4B/ATlxvCmw2N3XxvGfgJb5LWhm/c1srJmNXbBgQYbDFJEk/fpreNx7+/bwwgtw+eUwdWpIPNQuQ6Tiy1iyYWYHAvPdPbsky7v7IHfv6u5ds7KySjk6ESkP3OG556BDB7jySjjgAJgyJSQe9esnHZ2IlJZM9rOxG3CwmR0A1AEaAncDjc2sRqzdaAXMymAMIlJOjR8P55wDH3wAO+wAQ4bAnnsmHZWIZELGajbc/RJ3b+XurYG+wHvufiwwCjgyznYi8HKmYhCR8mf+fOjfH7p0gcmT4cEHITtbiYZIZZZEPxsXAeeZ2QxCG47BCcQgImVs9Wq44w5o2zbcWXLuuTB9ekg8qldPOjoRyaQy6a7c3UcDo+Prb4FuZbFdESkfRo6EgQNDPxn77w933hnaaYhI1aAeREUkY6ZMCY0+e/UK46+9Bq+/rkRDpKpRsiEipW7xYjjvPNhuO/joI7j9dpgwISQeIlL16KmvIlJq1q2DwYPhsstg0SLo1w+uvx422ijpyEQkSarZEJFS8b//hTtMTj8dtt463GEyaJASDREpQbJhZtXMrGEmghGRiuf776FPH9hrL/j5Z3jmmZB4dO6cdGQiUl6klWyY2TAza2hmGwATga/N7MLMhiYi5dmKFaHXzw4d4NVX4ZprQoPQPn3UxbiI/Fm6NRsd3X0p4aFprwNtgOMzFpWIlFvuMGxYeI7JddfBYYeF55hceSXUq5d0dCJSHqWbbNQ0s5qEZGOEu68BPHNhiUh5NHYs7L57eEBa8+ahq/Fhw2DTTZOOTETKs3STjQeBmcAGwPtmtjmwNFNBiUj5MncunHJKeNT7jBnhjpPPPw+Jh4hIUdK69dXd7wHuSSn63sz2zkxIIlJerFoFd98dLpesWgUXXhge/95QTcRFpBjSSjbMrClwFbA74fLJh8C1wKLMhSYiSXGHESPg/PPhm2/goINCx1xt2yYdmYhUROleRnkaWAAcQXhi6wLgmUwFJSLJmTQJevSAQw+FWrXgzTdD4qFEQ0RKKt1ko4W7X+fu38XheqB5JgMTkbL1889w1lmwww6hIeg998CXX4bEQ0RkfaSbbLxlZn1jh17VzKwP8GYmAxORsrF2Ldx3X6i5+O9/Qw+g06eHxKNmzaSjE5HKIN1k4zRgGLA6Dk8Dp5vZMjPTXSkiFdS770KnTnDmmaFG44svQuLRrFnSkYlIZZJWsuHuDdy9mrvXiEO1WNbA3dUuXaSC+eab0BnXvvvCypXw4osh8dhuu6QjE5HKKO2nvprZwcD/xdHR7v5qEfPXAd4HasftPO/uV5nZY8CewJI460nu/kVxAxeR4lu2DG68Ee64I1wiufFGGDgQ6tRJOjIRqczSvfX1ZmAnYGgsOsfMdnP3SwpZbBXQ3d2Xx95HPzSz1+O0C939+RJHLSLFkpMDTzwBF18cOug64QS46SbYZJOkIxORqiDdmo0DgE7ungNgZo8D44ECkw13d2B5HK0ZB3VxLlLGPvkEzjkn9PjZrRsMHw5/+1vSUYlIVVKcR8w3TnndKJ0FzKy6mX0BzAfedvcxcdINZvaVmd1pZrULWLa/mY01s7ELFiwoRpgiAjBrFhx/POy6K/z0EwwZEhIPJRoiUtbSTTZuAsab2WOxViMbuKGohdx9nbt3AloB3cxsW0JtSAfCZZkmwEUFLDvI3bu6e9esrKw0wxSRX3+FG26Adu3guefg0kth2rSQeFQrzr8XIiKlJN1nozxlZqMJCQLARe4+N92NuPtiMxsF7O/u/47Fq8zsUeCC4gQsIvlzD3eVXHABzJwJhx8Ot90GW2yRdGQiUtWl9X+OmRmwD6Hdxgiglpl1K2KZLDNrHF/XBfYDpphZi5R1HgpMXI/4RYTQ02f37nDkkdCgQbiN9YUXlGiISPmQbqXqf4FdgKPj+DLgviKWaQGMMrOvgM8JbTZeBYaa2QRgAtAMuL7YUYsIAAsWwBlnwI47woQJoQfQceNC4iEiUl6kezfK39x9RzMbD+Duv5hZrcIWcPevgM75lOs0KLKe1qwJicXVV4e+M848E666Cpo0SToyEZG/SjfZWGNm1Ym3rppZFpCTsahEpEBvvBE64poyJTwk7c47oWPHpKMSESlYupdR7gFeAjYysxuADwl3qIhIGZk2DQ48EHr2DA9Pe+WVkHgo0RCR8i7du1GGmlk2oZGoAYe6++SMRiYiACxZAtddFx75XqdOuMPkrLOgdr491IiIlD/pdld+qrsPBqaklN3s7hdnLDKRKm7dOnj00dBPxsKFcMopof+M5s2TjkxEpHjSbbNxhJn95u5DAczsPkCPbhLJkA8+CF2Mjx8Pu+0Gr78OXbokHZWISMmknWwAI8wsB9gfWOzup2YuLJGq6Ycf4F//gmeegVat4Kmn4KijwCzpyERESq7QZMPMUm+k6wcMBz4CrjGzJu7+cyaDE6kqVq6EW2+FW24J41ddFZKOevWSjUtEpDQUVbORTbjd1VL+9oqDA+qfUGQ9uIdajAsvDA9LO+qokHRstlnSkYmIlJ5Ckw13b1NWgYhUNdnZoV3GRx9B584wbBjssUfSUYmIlD49A1KkjM2dC6eeCjvtFPrOeOgh+PxzJRoiUnml20BURNbT6tWhr4xrrw2PgT/vPLjiCmjUKOnIREQyq6gGojXdfU1ZBSNSGbnDa6+F5GL6dOjVC26/Hdq3TzoyEZGyUdRllE/MbLiZ/cPMWpdBPCKVyuTJoXvxgw6CatVg5Eh49VUlGiJStRSabLh7V+DcOHqXmX1uZneaWQ8zU2fJIgX45Rc491zYbjv49NPwsLQJE0LiISJS1RTZQNTdZ7r7A+5+KLAr8AqwL/CBmb2W6QBFKpK1a+H++6FtW7j3XujXL1w6OfdcqFkz6ehERJJRrAaisf3Ge3HAzFpmIiiRimjUqHAr64QJsOeecPfdsMMOSUclIpK89br11d1nFTbdzOqY2Wdm9qWZTTKza2J5GzMbY2YzzOwZM6u1PnGIJOm77+CII6B7d1i6FJ5/PiQeSjRERIJM97OxCuju7jsAnYD9zWxn4BbgTnffCvgF0HNWpMJZvhwuuwy23hreeAOuvz40CD3iCD3LREQkVUaTDQ+Wx9GacXCgO/B8LH8cODSTcYiUppwceOIJaNcObrwRevcOnXNddhnUrZt0dCIi5U9abTbMrB1wIbB56jLu3j2NZasTnrGyFXAf8A3hqbFr4yw/AX9p+2Fm/YH+AJvpQRFSTowZE9pljBkTegB94QXYZZekoxIRKd/SbSD6HPAA8BCwrjgbcPd1QCczawy8BHRIc7lBwCCArl27enG2KVLaZs+GSy6BIUNg443hscfg+OND3xkiIlK4dJONte5+//psyN0Xm9koYBegsZnViLUbrYBCG5qKJOW330IfGTfcAGvWwMUXw6WXQoMGSUcmIlJxFPp/mZk1MbMmwCtmNsDMWuSWxfJCmVlWrNHAzOoC+wGTgVHAkXG2E4GX1+tdiJQyd3jpJejYMSQX++0HX38NN92kRENEpLiKqtnIJjTozG1bf2HKNAe2KGL5FsDjsd1GNeBZd3/VzL4Gnjaz64HxwOBiRy6SIRMmhE643nsPttkG3n4b9t036ahERCquQpMNd2+zPit396+AzvmUfwt0W591i5S2hQvhyivhwQehcWO47z7o3x9q6NnIIiLrJa3mbWb2z9zLIXF8QzMbkLmwRMrOmjXh0e9t28KgQTBgQOhifMAAJRoiIqUh3bb0p7n74twRd/8FOC0zIYmUnbfeCj19nnMOdOkCX3wRnmnSpMgWSSIikq50k43qZn/0iRjbYKiLcamwZsyAQw6Bv/8dVq2C4cND24xtt006MhGRyifdZONN4Bkz28fM9gGeAt7IXFgimbF0KVx0UbjL5L334Oabw10mhxyiLsZFRDIl3SvS/yL05nlGHH8beDgjEYlkQE5O6DWuH9QAACAASURBVIjr0kth3jw4+eTQd0aLFklHJiJS+RWZbMRLJkPc/VhCL6IiFcpHH4U2GdnZoWvxV14JXY2LiEjZKPIySuxufHM9Bl4qmh9/hGOOgd13h7lzYejQkHgo0RARKVvpXkb5FvjIzEYAK3IL3f2OjEQlsh5WroR//zu0x3CHK64I7TQ22CDpyEREqqZ0k41v4lANUGfNUi65w3PPwYUXwg8/hEe/33ortG6ddGQiIlVbWsmGu18DYGb14/jyTAYlUlzjx4d2GR98EPrNGDIE9twz6ahERATS70F0WzMbD0wCJplZtpltk9nQRIo2f37oUrxLF5g8OXQ1np2tRENEpDxJt5+NQcB57r65u28OnA88lLmwRAq3ejXccUfoYvzRR8OD06ZPD4lH9epJRyciIqnSbbOxgbuPyh1x99FmpuZ2koiRI2HgQJg2DXr2DElHhw5JRyUiIgVJt2bjWzO7wsxax+Fywh0qImVmypSQXPTqFcZfey0kHko0RETKt3STjVOALOBF4AWgGXBypoISSfXbb3DBBbDddvDxx6EmY8IEOOCApCMTEZF0pHsZZV93Pzu1wMx6A88VtICZbQoMAZoDDgxy97vN7GrCE2MXxFkvdfeRxQ1cqobvvgu3sGZnQ79+oYvxjTZKOioRESmOdJONS/hrYpFfWaq1wPnuPs7MGgDZZvZ2nHanu/+7eKFKVfPaa3D88eG5Ji+/DAcfnHREIiJSEoUmG2bWEzgAaGlm96RMakhIJgrk7nOAOfH1MjObDLRcv3ClKli3Dq68Em68ETp1guefhy23TDoqEREpqaLabMwGxgK/Adkpwwjg7+luxMxaA52BMbHoTDP7ysweMbMNixmzVGLz5kGPHiHR6NcvtNFQoiEiUrEVWrPh7l8CX5rZL8Cr7p5T3A3EXkdfAM5196Vmdj9wHaEdx3XA7YQGqHmX6094rD2bbbZZcTcrFdCHH8JRR8HPP4e+M046KemIRESkNKR7N0ofYLqZ3Wpmad9oaGY1CYnGUHd/EcDd57n7upi4PAR0y29Zdx/k7l3dvWtWVla6m5QKyD3cYbLXXlCvHnz6qRINEZHKJK1kw92PI1wG+QZ4zMw+MbP+seFnvszMgMHA5NSnw5pZi5TZDgMmlihyqRSWLg13m5x/fmgAOnZseLaJiIhUHunWbODuS4HngaeBFoREYZyZnVXAIrsBxwPdzeyLOBwA3GpmE8zsK2BvYOB6vQOpsCZMgK5dYfjw8Ej4F16ARo2SjkpEREpbWre+mtnBhE68tiL0ndHN3eebWT3ga+DevMu4+4eA5bM69akhPP44nHEGNG4Mo0bBHnskHZGIiGRKuv1sHEHoG+P91EJ3X2lmp5Z+WFJZ/fYbnHUWPPww7L03DBsGG2+cdFQiIpJJaSUb7n6imW0cazgc+Nzd58Zp72YyQKk8vv0WjjwSxo+HSy6Ba6+FGummuyIiUmGl1WYj1l58BhwOHAl8amZ/uV1VpCCvvAJduoTux195JfSjoURDRKRqSPd0/y+gs7svAjCzpsDHwCOZCkwqh7Vr4Yor4OabYccdQ2+gbdokHZWIiJSldJONRcCylPFlsUykQHPnwtFHw+jR0L8/3H031KmTdFQiIlLW0k02ZgBjzOxlQpuNQ4CvzOw8gNR+NEQAPvgg9Aa6eHG48+SEE5KOSEREkpJusvFNHHK9HP8W2KmXVE3ucPvtcPHFsMUW8OabsN12SUclIiJJSvdulGsyHYhUfEuWhG7Ghw+HI46ARx6Bhg2TjkpERJKWdg+iIoX58stwt8mrr4bnnDz3nBINEREJlGzIenv0Udh5Z/j119AYdOBAsPz6jhURkSpJyYaU2K+/Qr9+cMopsOuuobOu3XZLOioRESlv0u3Uq52ZvWtmE+P49mZ2eWZDk/Lsm29CgjF4MFx2Gbz1Fmy0UdJRiYhIeZRuzcZDwCXAGgB3/wrom6mgpHx7+eXQPuP770Mbjeuvh+rVk45KRETKq3STjXru/lmesrWlHYyUb2vXwr/+BYceClttBePGQa9eSUclIiLlXbr9bCw0sy0JHXphZkcCczIWlZQ7c+ZA377w/vvwj3/AnXeqN1AREUlPusnGP4FBQAczmwV8BxyXsaikXPnf/0JvoMuWwRNPwHH65EVEpBjSuozi7t+6+75AFtDB3Xd395mFLWNmm5rZKDP72swmmdk5sbyJmb1tZtPj3w3X+11IRrjDLbdA9+7QuDGMGaNEQ0REii+tmg0zawycALQGaljsRMHdzy5ksbXA+e4+zswaANlm9jZwEvCuu99sZhcDFwMXlfgdSEYsXgwnnggjRkDv3uGukwbqnF5EREog3csoI4FPgQlATjoLuPscYrsOd19mZpOBloSHuO0VZ3scGI2SjXJl/Hg48kj44Qe46y44+2x10iUiIiWXbrJRx93PK+lGzKw10BkYAzSPiQjAXKB5Acv0B/oDbLbZZiXdtBTT4MHwz39Cs2ahMeguuyQdkYiIVHTp3vr6hJmdZmYtYpuLJmbWJJ0Fzaw+8AJwrrsvTZ3m7k68wyUvdx/k7l3dvWtWVlaaYUpJrVwZegLt1w/22CPUbijREBGR0pBusrEauA34BMiOw9iiFjKzmoREY6i7vxiL55lZizi9BTC/uEFL6ZoxIyQWjz4KV1wBb7wByu9ERKS0pHsZ5XxgK3dfmO6KLbQiHQxMdvc7UiaNAE4Ebo5/X053nVL6XnopPBa+Rg0YORJ69kw6IhERqWzSrdmYAaws5rp3A44HupvZF3E4gJBk7Gdm04F947iUsTVr4MIL4fDDoX370BuoEg0REcmEdGs2VgBfmNkoYFVuYWG3vrr7h0BB9zDsk3aEUupmzw69gX7wAQwYAHfcAbVrJx2ViIhUVukmG8PjIBXcqFFw9NGhN9ChQ+GYY5KOSEREKru0kg13fzzTgUhm5eSE3kAvvxzatoV334Vttkk6KhERqQoKTTbM7Fl372NmE8jnFlV33z5jkUmp+eUXOOGE8Dj4o46Chx5Sb6AiIlJ2iqrZOCf+PTDTgUhmjBsXegP96Se45x4480z1BioiImWr0LtRUnr6HODu36cOwIDMhycl5Q6DBsGuu8LataE30LPOUqIhIiJlL91bX/fLp0w3SpZTK1eGvjNOPx323DPUbuy8c9JRiYhIVVVUm40zCDUYW5jZVymTGgAfZTIwKZlp08Jlk4kT4aqrQo+g1asnHZWIiFRlRbXZGAa8DtxEeBR8rmXu/nPGopISeeEFOPlkqFULXn8d/v73pCMSEREpItlw9yXAEuDosglHSmLNGrjoIrjzTvjb3+DZZ0EPyhURkfIi3U69pJyaNSvczvrRR+FOk9tvDzUbIiIi5YWSjQrsvfdCb6ArVsBTT4UuyEVERMqbdO9GkXIkJwduvBH22w+aNoXPP1eiISIi5ZdqNiqYn38OvYG+9lqo1Rg0COrXTzoqERGRginZqEDGjoXevUM7jf/8JzyxVZ10iYhIeafLKBWAOzzwAOy2G6xbFx4N/89/KtEQEZGKQclGObdiRbhscsYZ0L07jB8fbm8VERGpKDKabJjZI2Y238wmppRdbWazzOyLOByQyRgqsqlTQ2IxdChcc01op9G0adJRiYiIFE+mazYeA/bPp/xOd+8Uh5EZjqFCeu456NoV5s6FN96AK6+EaqqHEhGRCiijP1/u/j6gbs2LYfVqOPdc6NMHtt02XDbp0SPpqEREREouqf+VzzSzr+Jllg3zm8HM+pvZWDMbu2DBgrKOLxE//QR77QV33w3nnAP/+x9sumnSUYmIiKyfJJKN+4EtgU7AHOD2/GZy90Hu3tXdu2ZlZZVlfIl45x3o3BkmTIBnnoG77lK34yIiUjmUebLh7vPcfZ275wAPAd3KOobyJCcHrrsuXCpp3jz0BtqnT9JRiYiIlJ4y79TLzFq4+5w4ehgwsbD5K7NFi+D448Pj4I89Fh58EDbYIOmoRERESldGkw0zewrYC2hmZj8BVwF7mVknwIGZwOmZjKG8+vxzOPLIcLfJ/ffD6aerky4REamcMppsuPvR+RQPzuQ2y7vc3kDPPRdatIAPP4Sddko6KhGRzMnOzt6oRo0aDwPbos4kK6scYOLatWv7denSZX7eiXo2ShlasSLUYAwdCj17whNPqJMuEan8atSo8fDGG2+8dVZW1i/VqlXzpOOR0peTk2MLFizoOHfu3IeBg/NOV4ZZRqZMgW7dYNiw0CD01VeVaIhIlbFtVlbWUiUalVe1atU8KytrCaH26i9Us1EGnnkGTj0V6tWDt96CffdNOiIRkTJVTYlG5Rc/43wrMVSzkUGrV8PZZ0PfvrDDDjBunBINERGpepRsZMiPP8L//R/ce29oDDp6NLRqlXRUIiKSjosvvnjj0lrXeeedt8mVV17ZvKDpxxxzzGZvvfXWBt26dWv//vvv1yut7Zam+++/v0m7du06tmvXrmPnzp07fPLJJ3WLs7ySjQx4663QG+ikSfDss3DnnVCzZtJRiYhIuu65554WxZk/JyeHdevWlWhb48aNq9+9e/cVJVq4jGy11VarPvroo6nTpk37+pJLLpl9+umnb16c5ZVslKKcnPAo+P33h403hrFjoXfvpKMSEanapk6dWqtNmzbbHHzwwW222GKLbfbff/8tli1bVm3EiBEN9t133y1z53vppZca7rffflsOGDCg5apVq6p16NCh48EHH9wG4Oqrr27etm3bbdq2bbvNtddeu1Huelu3br3tYYcd1rpdu3bbfPPNN7Wef/75hh07dty6ffv2HXfZZZd2ueuePHly3W7durVv1arVdtdff/1GueXjxo2rs8UWW/xWo8YfTSjXrVvHEUcc0frss8/eBODFF19s2KlTpw4dO3bcumfPnlssWbKkGsDLL7/cYOutt+7Yrl27jr17927966+/GkDLli23+8c//tGqXbt2HbfbbrutJ06cWBvgkUce2bBt27bbtG/fvmPXrl3bF2cf7rfffiuysrLWAey9994r5s6dW6wHaqiBaClZuBCOOw7efDP0Cnr//eoNVEQkr1NOYdOJEynVSwXbbsvKRx7hx8LmmTlzZp0HH3xwZo8ePVb07t279W233ZZ19dVXzzvnnHM2mz17do1NNtlk7SOPPNL05JNPXnjMMccseeyxxzaaMmXK1wAffPBBvWHDhjXNzs6e7O506dJl63322WdZs2bN1v3www+1Bw8e/N0+++wzc/bs2TXOPPPM1qNHj57SoUOH1fPmzaueu/0ZM2bU+fjjj6cuXry4+tZbb73thRdeuKB27do+YsSIRj169FiSO9+aNWvs0EMPbdOxY8dfb7nllrlz5sypceONN7Z4//33pzVs2DDnsssu2/i6665rfu211849/fTT27z11ltTt99++1WHHXZY69tuuy3ryiuvnA/QqFGjtdOmTfv6P//5T9Ozzjpr01GjRs24+eabW7z11lvT2rRps2bhwoXV/7qX0nPvvfc223vvvZcUPecfVLNRCsaMgR13hFGjQoddjz+uRENEpDzZeOONV/fo0WMFwPHHH7/o448/rl+tWjX69Omz6KGHHmqycOHC6uPGjavfu3fvv/yIjh49uv4BBxywuGHDhjmNGjXK6dWr1y+jRo1qANCiRYvV++yzz4o43wbdunVb1qFDh9UAzZs3//26So8ePRbXrVvXW7RosbZJkyZrfvrppxoA77zzTsNDDz10ae58AwYM2Dw30chd5zfffFOnW7duHTp06NDx6aefbvrDDz/U+vLLL+u0atVq1fbbb78K4KSTTlr04YcfNshdz4knnvgzwGmnnfbz+PHj6wN07dp1+bHHHtv69ttvb7Z27doS7cdXXnmlwZNPPtns7rvv/qk4y6lmYz24w333wXnnQcuW8PHH0KVL0lGJiJRfRdVAZIrleR5E7vgZZ5yxqFevXlvVqVPHDzrooF9qFrOBXb169XLSma927dq/3/pbvXp11q5da8uWLau2dOnS6q1bt16TO61r167LP/jgg4YrV66cV69ePXd3dt9996WvvPLKd6nrK6qBZrVqf9QlmJkDDBs27If33ntvgxEjRjTq0qVLx+zs7K833njj3xOis846q+Xbb7/dCCC3VifVmDFj6g4YMGDz1157bXrqculQzUYJLV8OxxwDZ50Vntiana1EQ0SkvJozZ06td955ZwOAoUOHNtl1112XA7Ru3XpN8+bN19x+++0t+vfvvzB3/ho1aviqVasMYO+9914+cuTIxjE5qDZy5MgN995772V5t7HXXnut+OyzzxpMmTKlFkDqZZT8vPbaaw123333P63n9NNPX9ijR48lBx544JZr1qxhr732WjF27Nj6ue0uli5dWu2rr76qvcMOO/w2a9asWrnlQ4YMabrHHnv8vq4hQ4Y0ARg8ePCGnTt3XgEwadKk2t27d19x1113zd5www3Xfvvtt39qd3HvvffOmjJlytf5JRrTp0+v1bt37y0feeSR73JrU4pDNRslMHkyHHEETJ0KN94IF10E1ZS2iYiUW61bt/7t3nvv3ah///712rZt+9sFF1ywIHda3759F9133301dtxxx99yy4499tgFW2+9dcdtt9125YgRI7475phjFu24445bAxx//PELdtttt1+nTp36px/rTTbZZO0999wz87DDDtsqJyeHpk2brvn444+nFxTTyJEjG/Xp0+eXvOVXX331vIEDB1Y//PDD2wwfPvy7Bx98cGbfvn23WL16tQFcddVVs7bffvtVDzzwwMzevXtvuW7dOnbYYYeVqe/pl19+qd6uXbuOtWrV8qeffvpbgIEDB7aaOXNmbXe33XfffenOO+/8a7r77/LLL2+xePHiGmedddbmEJKxiRMnTk53eXMv/526de3a1ceOHZt0GAA89RScdlpok/HUU9C9e9IRiYjkz8yy3b1r0nF8+eWXM3fYYYeFRc+ZGVOnTq114IEHtp0+ffqk/KafcMIJm3Xu3HnlwIEDyzTGjh07bj1+/PgpqZdYSkPLli23Gzt27OQWLVqUrGHGevjyyy+b7bDDDq3zlqtmI02rVsH554c2GrvtFrogb9ky6ahERGR9bLPNNlvXrVs358EHHyzztiRff/112jUDFZ2SjTR8/z306QOffRYag958szrpEhGpKNq3b7+6oFqNSZMmVbof/FmzZk1IOoa8lGwU4Y034NhjYc0aeOEFOPzwpCMSEalwcnJyckwPY6vccnJyDMj37pyMNms0s0fMbL6ZTUwpa2Jmb5vZ9Ph3w0zGUFLr1sFVV8EBB4TLJdnZSjREREpo4oIFCxrFHyOphHJycmzBggWNgIn5Tc90zcZjwH+AISllFwPvuvvNZnZxHL8ow3EUy4IFoTbj7bfhxBPhv/8Nj4cXEZHiW7t2bb+5c+c+PHfu3G1RlwuVVQ4wce3atf3ym5jRZMPd3zez1nmKDwH2iq8fB0ZTjpKNTz4J7TMWLIBBg6BfPzDl4iIiJdalS5f5wMFJxyHJSSLDbO7uc+LruUC+j901s/5mNtbMxi5YsCC/WUqVO9xzT3gsfM2aoTfQ005ToiEiIrK+Eq3O8tDJR74Nhtx9kLt3dfeuWVlZGY1j2TLo2xfOOQd69gztM3bcMaObFBERqTKSSDbmmVkLgPh3fgIx/G7SJNhpJ3j+ebjpJhg+HDYsl01WRUREKqYkko0RwInx9YnAywnEAMDQodCtG/zyC7zzDlx8sbodFxERKW2ZvvX1KeAToL2Z/WRmpwI3A/uZ2XRg3zheplatggED4LjjwsPTxo+Hvfcu6yhERESqhkzfjXJ0AZP2yeR2C/P999C7N3z+OVx4YXiQWg11bSYiIpIxVepnduTIUJuxbh289BIcemjSEYmIiFR+VaKFwrp1cMUV0KsXbLZZuNtEiYaIiEjZqPQ1G/PnwzHHwLvvwsknh6e21q2bdFQiIiJVR6VONiZMCP1mLFoEgwfDKackHZGIiEjVU6kvo2y6KWyzTeiCXImGiIhIMip1zUbjxvDmm0lHISIiUrVV6poNERERSZ6SDREREckoJRsiIiKSUUo2REREJKOUbIiIiEhGKdkQERGRjFKyISIiIhmlZENEREQyytw96RiKZGYLgO/XYxXNgIWlFE5pUlzFo7iKR3EVT2WMa3N3zyrNYERKokIkG+vLzMa6e9ek48hLcRWP4ioexVU8ikskc3QZRURERDJKyYaIiIhkVFVJNgYlHUABFFfxKK7iUVzFo7hEMqRKtNkQERGR5FSVmg0RERFJiJINERERyagKmWyY2aZmNsrMvjazSWZ2TixvYmZvm9n0+HfDWG5mdo+ZzTCzr8xsx5R1nRjnn25mJ65nXHXM7DMz+zLGdU0sb2NmY+L2nzGzWrG8dhyfEae3TlnXJbF8qpn9fX3iSllndTMbb2avlpe4zGymmU0wsy/MbGwsS/RzjOtrbGbPm9kUM5tsZrskHZeZtY/7KXdYambnJh1XXN/AeMxPNLOn4nehPBxf58SYJpnZubGszPeXmT1iZvPNbGJKWanFYWZd4vdoRlzWSrbHRDLE3SvcALQAdoyvGwDTgI7ArcDFsfxi4Jb4+gDgdcCAnYExsbwJ8G38u2F8veF6xGVA/fi6JjAmbu9ZoG8sfwA4I74eADwQX/cFnomvOwJfArWBNsA3QPVS2G/nAcOAV+N44nEBM4FmecoS/RzjOh8H+sXXtYDG5SGulPiqA3OBzZOOC2gJfAfUTTmuTkr6+AK2BSYC9YAawDvAVknsL+D/gB2BiZk4zoHP4rwWl+1ZGseZBg2lNSQeQKm8CXgZ2A+YCrSIZS2AqfH1g8DRKfNPjdOPBh5MKf/TfOsZUz1gHPA3Qu9/NWL5LsCb8fWbwC7xdY04nwGXAJekrOv3+dYjnlbAu0B34NW4nfIQ10z+mmwk+jkCjQg/nlae4soTSw/go/IQFyHZ+JHwI1gjHl9/T/r4AnoDg1PGrwD+ldT+Alrz52SjVOKI06aklP9pPg0aysNQIS+jpIpVsJ0JtQjN3X1OnDQXaB5f554Mc/0UywoqX594qpvZF8B84G3Cf2eL3X1tPtv4fftx+hKgaSbiAu4inGhz4njTchKXA2+ZWbaZ9Y9lSX+ObYAFwKMWLjs9bGYblIO4UvUFnoqvE43L3WcB/wZ+AOYQjpdskj++JgJ7mFlTM6tHqDHYlPLzOZZWHC3j69KOT6TUVOhkw8zqAy8A57r70tRp7u6EH7Iy5e7r3L0ToSahG9ChrGPIy8wOBOa7e3bSseRjd3ffEegJ/NPM/i91YkKfYw1Clff97t4ZWEGo5k46LgBi24eDgefyTksirtjW4BBCkrYJsAGwf1nGkB93nwzcArwFvAF8AazLM09in2N5jEMkUypssmFmNQmJxlB3fzEWzzOzFnF6C0LtAsAswn80uVrFsoLK15u7LwZGEaqPG5tZjXy28fv24/RGwKIMxLUbcLCZzQSeJlxKubscxJX7XzHuPh94iZCgJf05/gT85O5j4vjzhOQj6bhy9QTGufu8OJ50XPsC37n7AndfA7xIOObKw/E12N27uPv/Ab8Q2nclvb9ylVYcs+Lr0o5PpNRUyGQjtrQeDEx29ztSJo0Aclton0hoy5FbfkJs5b0zsCRWX74J9DCzDeN/Zz1iWUnjyjKzxvF1XUI7ksmEpOPIAuLKjfdI4L34H84IoG9std8GaEtoAFYi7n6Ju7dy99aE6vf33P3YpOMysw3MrEHua8L+n0jCn6O7zwV+NLP2sWgf4Ouk40pxNH9cQsndfpJx/QDsbGb14nczd38lenwBmNlG8e9mwOGEBtJJ769cpRJHnLbUzHaO+/+ElHWJlA9JNxopyQDsTqhy/IpQNfoF4XpsU0IjyOmEludN4vwG3EdoPzEB6JqyrlOAGXE4eT3j2h4YH+OaCFwZy7cgnDRnEKq+a8fyOnF8Rpy+Rcq6LovxTqUUW5YDe/HH3SiJxhW3/2UcJgGXxfJEP8e4vk7A2PhZDie0/i8PcW1AqAVolFJWHuK6BpgSj/snCHeUJH7cAx8QEp8vgX2S2l+E5HAOsIZQc3ZqacYBdI37/hvgP+Rp3KxBQ9KDuisXERGRjKqQl1FERESk4lCyISIiIhmlZENEREQySsmGiIiIZJSSDREREckoJRtSqZnZOgtPSP3SzMaZ2a5FzN/YzAaksd7RZta1hDGNzO2PRUSkKlCyIZXdr+7eyd13IDzo66Yi5m9MeCppxrj7AR56mBURqRKUbEhV0pDQZTVmVt/M3o21HRPM7JA4z83AlrE25LY470Vxni/N7OaU9fU2s8/MbJqZ7ZF3Y2bWwszej+uamDuPmc00s2Zm9o847Qsz+87MRsXpPczskxjbcxaeASQiUmGpUy+p1MxsHaEXxjqER3F3d/fs+EyOeu6+1MyaAZ8SusfenNDD6rZx+Z6ER5Pv6+4rzayJu/9sZqOBbHc/38wOAM5z933zbPt8oI6732Bm1eP2lll4Rk1Xd18Y56sJvAfcCnxCeLZIT3dfYWYXEXrevDaT+0lEJJNqFD2LSIX2q4en8GJmuwBDzGxbQpfQN1p4ymwO4ZHczfNZfl/gUXdfCeDuP6dMy30AYDbQOp9lPwceicnEcHf/ooAY7yY8H+QVC0/o7Qh8FB5zQS1CAiIiUmEp2ZAqw90/ibUYWYRn6WQBXdx9TaxtqFPMVa6Kf9eRz3fJ3d+PyUwv4DEzu8Pdh6TOY2YnEWpTzswtAt5296OLGYuISLmlNhtSZZhZB6A68UFmwPyYaOxN+MEHWAY0SFnsbeBkM6sX19GkGNvbHJjn7g8BDxMeU586vQtwAXCcu+fE4k+B3cxsqzjPBmbWrnjvVESkfFHNhlR2dc0s9/KFASe6+zozGwq8YmYTCE93nQLg7ovM7CMzmwi87u4XmlknYKyZrQZGApemvNaUIQAAAGtJREFUue29gAvNbA2wnPDo71RnAk2AUfGSyVh37xdrO54ys9pxvsuBacV+5yIi5YQaiIqIiEhG6TKKiIiIZJSSDREREckoJRsiIiKSUUo2REREJKOUbIiIiEhGKdkQERGRjFKyISIiIhn1/2p/esEbfqUvAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhsAAAEWCAYAAADPUVX+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd5gUVdbH8e8hiwQFERFFUCQpIsJiwndFlBV0FVdhWRMqioo5rbq65uyadRUDYo6rgoouSjCjEkSiCoooGSSDpDnvH/eONrMTeobpqQm/z/P001230unq6urTt27dMndHREREJFMqJR2AiIiIlG9KNkRERCSjlGyIiIhIRinZEBERkYxSsiEiIiIZpWRDREREMqrMJhtm5mbWPJ/xj5jZP1OGzzazBWa2yszqF2F9TeM6qxQ15uIS38OuScdRUsxssJndVIjp8903Shsze8fM+iYdRzrMrEnc/yoXYd6tzOxNM1tuZq9kIj4pGjM7wcyGJx2HlF8FJhtmNsvM1pvZdjnKJ8SDetM4PDgOH51junti+Slx+BQz2xQPWKvM7Acze9LMWhTbuwLc/Sx3vzGusypwN9DN3Wu5+5JM/iDF9/hxMS1rtJmdnloW38P3xbH84lAaEjEz29/MPk1q/ekys+vM7NnUMnfv7u5PJRVTfuL3/9DsYXefHfe/TUVY3HFAQ6C+u/cqhtiqmdmrMUY3s4NzjDczu93MlsTH7WZmKeP3NrNxZrYmPu+9pTElobDHm9y+r+7+nLt3y0yEIunXbPwA/C17wMzaAjVzme5b4OSU6aoAvYGZOab7zN1rAXWBQ4G1wDgz2zP90POWy7+uhkANYEpxLF+KVzElKUcAw4phORVGAsnhLsC37r6xsDPmE+vHwInA/FzG9Qd6Au2AvYA/A2fG5VUDhgDPAtsCTwFDYnnGxASozNYoixSZu+f7AGYBVwNfppT9C7gKcKBpLBscyxcA28ayI4F3CAeEU2LZKcDHuaznLeDVfOK4DJgHzAVOi+tunrLuhwk/NqsJCcxg4CagRSxzYBUwEvgwDq+OZX/NZX2V4/tZDHwPnBPnqRLH1wWeiDHNieuqDLQGfgU2xWUvi9NXj8ubHbfRI8BWKes7GvgKWEFIzg4Hbo7L+TUu68E4bep7rws8DSwCfoyfVaXUbR3Xu5SQNHYv4LO+Epgap38SqBHHTQb+nDJt1bht2sf3lL19VwH7ExLZq2NMC2OMdeO8TeP0/eK8H8byzsCnwDLgJ37fZwYDDwFvAyuBz4HdcsQ+Htgnl+3TOS7r4DjcCngP+AX4BuidsoyCtuUnwIPAcmA60DVl3lMI+8nKuJ1PyGX7Hg6sBzbE7TQxlo8GTs+xnnvidvgeOCCW/xS3Zd+UZea7X+VYf+qylxD22d0I34kl8fN8DtgmTv8MkEX4M7AK+HvKZ5f9PdgRGBq35wzgjDzWfX2O996PIuwj+ey7P2d/xillnwL9U4b7AWPi626E762ljJ8NHJ7H8kcDtwJfEL6jQ4B6KeP34/d9d2JqLHHem+O2Xws0j2U3xXlWAW8C9eP2XwF8ye/H1s22eeo+Q97HmyOACXFZPwHX5XifOb+vp5ByXCbsc18S9vUvgQNyrPvG+H5WAsOB7eK4GoQEbkncFl8CDQv6ndGj/D8KniD8AB1KODC3Jvyg/kz4l5Iz2bgJeBQ4O5a9TKgRSSfZOA1YkEcMhxMOpHsCWwPP87/JxnLgQMIBrEZ2PHF8bl/W3+bPY51nEX5QdgbqAaPY/CD7OjAwxrM94SB0Zl7vkXCAHxqXVZtwcLk1jusU4z8sxt8YaBXHjSb+EOUWO+EAPSQusymhdqlfShwbgDPi53Y2IVmzPN7zLEJSkf2eP0nZhn8HXkqZ9mhgUj7b9zTCj8+uQC3gNeCZHNM/HbffVoT9aWXcX6oSDrx7p3y+S+J2qkI4IL+Ysq5GpPxwZG8fwn7zE9Aplm8dh0+Ny2lP+IFtk+a23AhcFOP7a/zM6sXlrgBapsSzRx7b+Drg2Rxlv33GKes5NX5mNxF+HB4iJBbd4naqVdB+lcu6s5d9Xnz/W8XtdFhcdgNCIn5vzu9/yvBmn3Wc/t+E79zehETtkHTeO4XcRwo4TuWWbCwH9k0Z7gisjK8vAt7JMf1bwCV5LH80YR/LPgb9J/u9EL6vS4AehO/vYXG4Qcq8s4E94navGstmEJK9uoQE/1vCsbZKfN9P5vP9yrnP5DzeHAy0jfHsRTh+9sxneb8tg7AvLQVOirH8LQ7XT1n3TMIfua3i8G1x3JmEfbAmYf/tANQp6HdGj/L/KEx13jOEUySHAdMIX7zcPA2cbGbbAH8E3khz+XMJO3luehO+eJPdfTXhoJXTEHf/xN2z3P3XNNeZn96Eg+5P7v4L4V8NAGbWkHBgudDdV7v7QsJBv09uC4rnifsDF7n7L+6+ErglZfp+wCB3fy/GP8fdpxcUYDxd1Ae40t1Xuvss4C7CQSLbj+7+mIdz7E8Rfggb5rPYB1Pe8838fvrsWaCHmdWJwycR9om8nADc7e7fu/sqQo1JnxzV4dfF7bcWOB54391fcPcN7r7E3b9KmfZ1d//CQxX8c4Qftmw9gHfd3VPKehGSwe7u/kUsOxKY5e5PuvtGd59A+NHolea2XEjYJza4+0uEBPyIOC4L2NPMtnL3ee6+JafsfogxbgJeIiR/N7j7OncfTqghaJ7GfpWbue7+QHz/a919Rtzv1rn7IkLbpj+mE6SZ7UxI8C9391/j5/U4KadSC1DYfaSwahESjmzLgVpxu+Uclz2+dj7LeyblGPRPoHfcb04Ehrn7sPj9fQ8YS9gvsw129ylxu2+IZU+6+0x3X06oAZ7p7u/HffwVQjJcJO4+2t0nxXi+Bl4gzc+VsE9/5+7PxHhfIPzx+nPKNE+6+7fxc3mZ37+PGwh/FJq7+yZ3H+fuK4r6PqT8KMw522cI/2KaERKKXLn7x2bWgHCa5S13X5vSJis/jQlVsbnZERiXMvxjLtP8lM5KCmHHHMtMXecuhH8n81LeW6V8YmhAyPTHpbZPI2T+EH5MitLeYLsYR2psPxK2ZbbfzmW7+5q4/lr5LDPne94xzjvXzD4BjjWz14HuwAX5LGfHXOKqwuaJTuq6duZ/2/akSj0nv4bN30MPQm1XqguBp919ckrZLsC+ZrYspawKYd9OZ1vOyZHQ/Ajs6O6rzeyvwKXAE3E7XZJOwpiHBSmv1wK4e86yWhS8X+Vms300Js73AQcRfmgrEf7FpmNHIDvJyfYjoQYh3fkLs48U1iqgTspwHWCVu7uZ5RyXPX4lecv53ahK2G92ISSsqT/GVQm1obnNmy3nZ5rbZ1wkZrYvcBuhJqYaoeYq3SuAcn4ukM9xhc2/j88Qvssvxj+czwJXpSRYUkGlXbPh7j8SzkX3IFR35udZ4BLySUpycQzwUR7j5hF24GxNcguxEOtKR37r/AlYRzhPuU181HH3PfKIZTHh4LFHyvR1PTSSzV7ebnnEkd/7Wkz4J7FLjjjzqnVKR873PDdl+CnCv7hehEa+2evJLca5ucS1kc0PqKnz5bcN8hSvNPojoR1Gql5ATzNLTYh+Aj5I+Qy28XBlxdmkty0b2+aZ82/bx93/6+6HEWqOpgOP5RFyce6nBe1X6az/lljW1t3rED5fy2f6VHOBemaWWhtQmP2vsPtIYU0hNA7N1o7fG4lPAfbK8XnuRf6NyHN+NzYQPoOfCLUeqfvV1u5+W8r0W/I+Vsfn1Eb5OxSw7OcJp9d2dve6hLY8ls/0qXJ+LpDm5xpr/a539zaEdh9Hkn5Nl5RjhW0V3Y9wPnZ1AdPdTzjd8mF+E5lZZTNrZmYPEM4xXp/HpC8Dp5hZGzOrCVxbuLBztYBwrjgvLwPnm9lOZrYtcEX2CHefR2gUdZeZ1TGzSma2m5llV1MuAHbKbtnu7lmEH597zGx7ADNrbGZ/itM/AZxqZl3jshqbWauC4ozV7C8DN5tZbTPbBbiYkOwV1TnxPdcj1E69lDLuDWAfQo1GaiK5iHAaITXOF4CL4udbi/Cj9pLnfSXCc8ChZtbbzKqYWf00L0XsDHydS1XtXKArcIGZnR3L3gJamNlJZlY1Pv5gZq3T3JbbE/aJqmbWi9CGaZiZNTSzo81sa0ISuipuj9wsAJoWxxUJaexX6ahNiHe5mTUmNMTOGW9e+99PhAaOt5pZDTPbi3CMSHf/K+w+8j/MrLqZ1YiD1WIc2T+qTwMXx22yI+EP0OA4bjShUeX5cRnnxvKR+azuxJRj0A2EBu2bCO/3z2b2p3hMq2FmB5vZTum+j/zE01tz4vorm9lpbJ6Yb3a8iWoTap1+NbNOhNOU2XL7vqYaRvieHB+/i38F2hC+P/kysy5m1jaeXlpBSMjy+i5IBVKoA148vzg2jel+cfcROaqcU+0fqzFXEL70dYA/uPukPJb3DnAv4UAwg/wPCOm6DnjKzJaZWe9cxj8G/JfQsnw8/1ubczKhejL7yo1XCf9qifFNAeab2eJYdnmMfYyZrQDeB1rG9/cFoUHgPYTzxh/w+z+L+4DjzGypmd2fS5znEf75fE9oiPs8MCi9TZCr5wmJ1PeE0xq/daYVz8/+h3Aq7bWU8jXE1vZxe+4XY8g+9fYDocX8eXmt1N1nE2rNLiGcTvuKzf+V5iXPS17jMrsCV5jZ6bG6vxuhTcNcQlXw7YQqZih4W34O7E74N3szcJy7LyF8jy6Oy/yFUNNyNrnLrspeYmbj03h/Bclzv0rT9YQEcjnhap+c+/mtwNXxc700l/n/RmhwOJfQaPpad38/zXUXah/JwzeE2p3GhO/rWn7/7gwkNFacRGj4/HYsw93XEy6LPZlw1cRphAaU6/NZ1zOEZGU+oUHs+XFZPxEaTP+D8EP+EyFpK85LXM+Iy1xCaGia2qdMbsebAcANZrYSuIaQSBPjze37Ssr4JYQaiUvi+v4OHOnuiynYDoRj4QpC274PyL9tl1QQlnc+IBWNmc0itHDP88fCzK4BWrj7iSUWWD7MbCrhR39qhtdzCmHbdM7keqR0MrPRhKtPHk86FpGyKPGut6XsiKdW+rH5FRqJidXGT2c60RARkS2jnuwkLWZ2BqF6+B13z7ctTklx9/U5GuGJiEgppNMoIiIiklGq2RAREZGMKhNtNrbbbjtv2rRp0mGIiJQp48aNW+zuDZKOQ6RMJBtNmzZl7NgCr7gVEZEUZpZbb8siJU6nUURERCSjlGyIiIhIRinZEBERkYxSsiEiIiIZpWRDREREMkrJhoiIiGSUkg0RERHJKCUbIiKlkDs8/TS88krSkYhsOSUbIiKlzLRp0KUL9O0Lzz6bdDQiW07JhohIKbF2LVx9NbRrBxMnwsCB8PrrSUclsuUy3l25mc0CVgKbgI3u3tHM6gEvAU2BWUBvd1+a6VhEREqrd9+Fc86B77+Hk06Cf/0Ltt8+6ahEikdJ1Wx0cfe93b1jHL4CGOHuuwMj4rCISIUzdy707g3du0PVqjByZGiroURDypOkTqMcDTwVXz8F9EwoDhGRRGzaBPffD61awdChcOON4dRJly5JRyZS/Eoi2XBguJmNM7P+sayhu8+Lr+cDDXPOZGb9zWysmY1dtGhRCYQpIlIyxo6FTp3gggtg//1h8uTQVqN69aQjE8mMkkg2Orv7PkB34Bwz+7/Uke7uhISEHOWPuntHd+/YoEGDEghTRCSzli+H884LicbcufDSS6GtRvPmSUcmklkZTzbcfU58Xgi8DnQCFphZI4D4vDDTcYiIJMU9JBatWsFDD8G558L06aGthlnS0YlkXkaTDTPb2sxqZ78GugGTgaFA3zhZX2BIJuMQEUnKjBlw+OHQpw80bgxffBHaatStm3RkIiUn05e+NgRet5C6VwGed/d3zexL4GUz6wf8CPTOcBwiIiVq3Tq44w64+WaoVg0eeADOPhsqV046MpGSl9Fkw92/B9rlUr4E6JrJdYuIJGXkyJBYfPttOFVyzz2w445JRyWSHPUgKiJSTBYsCB1yde0KGzfCO++EthpKNKSiU7IhIrKFsrJC1+KtWoXk4uqrw+Wshx+edGQipUPGuysXESnPJk6Es86CMWPg4IPh4YdD0iEiv1PNhohIEaxaBZdcAh06wMyZoYvxkSOVaIjkRjUbIiKF4A5DhoTOuX7+Gfr3h1tvhXr1ko5MpPRSzYaISJp+/BGOPhqOOQa23RY++SS01VCiIZI/JRsiIgXYsCH0mdGmDYwYEW7/Pm4cHHBA0pGJlA06jSIiko+PPw59ZkyeDD17wn33QZMmSUclUraoZkNEJBdLlsDpp8NBB4UbqA0ZAq+/rkRDpCiUbIiIpHCHwYOhZcvwfNllMHUqHHVU0pGJlF06jSIiEk2dGk6ZfPhhaI/xyCPQtm3SUYmUfarZEJEKb80a+Mc/oF07mDQJHnsMPvpIiYZIcVHNhohUaMOGwbnnwg8/QN++cOed0KBB0lGJlC+q2RCRCmnOHOjVC444AqpXh1GjQhsNJRoixU/JhohUKBs3hstXW7WCt96Cm28O9zc5+OCkIxMpv3QaRUQqjC++CDdNmzAh3JH1oYdg112Tjkqk/FPNhoiUe8uWwTnnwH77wYIF8Moroa2GEg2RkqFkQ0TKLXd44YVwyuSRR8LN06ZNg+OOA7OkoxOpOHQaRUTKpe++gwED4P33oWPHUJOxzz5JRyVSMalmQ0TKlXXr4PrrQx8ZX3wBDz4IY8Yo0RBJkmo2RKTcGDEi1GZ8+y306QN33w2NGiUdlYioZkNEyrwFC+CEE+DQQ2HTJvjvf0NbDSUaIqWDkg0RKbOyskLDz5Yt4dVX4ZprQnfj3bolHZmIpNJpFBEpk776KvSZ8fnncMgh8O9/h6RDREof1WyISJmyciVcfDF06BDuZ/Lss+GKEyUaIqWXajZEpExwh9dfh/PPh7lz4cwz4ZZbYNttk45MRAqimg0RKfVmzYI//xmOPRa22w4+/RQefliJhkhZoWRDREqt9evhttugTRsYPTpcyjp2bOh2XETKDp1GEZFS6aOPQgPQqVPhL3+Be++FnXdOOioRKQrVbIhIqbJ4MZx2Gvzf/8Hq1fDmm/Cf/yjRECnLlGyISKmQlQVPPhlumvbMM3D55TBlChx5ZNKRiciW0mkUEUnclClw9tnh1EnnzqHx5557Jh2ViBQX1WyISGLWrIErr4S99w4JxxNPwAcfKNEQKW9UsyEiiXj7bTj33HBZ66mnwh13hMtaRaT8Uc2GiJSon38O/WUceSTUrBlqMgYNUqIhUp4p2RCRErFxI9xzD7RuDe+8A7feChMmhKtORKR802kUEcm4zz8P3YtPnAg9esCDD0KzZklHJSIlRTUbIpIxy5bBgAGw//6waFG4DfxbbynREKloCp1smFklM6tTiOkrm9kEM3srDjczs8/NbIaZvWRm1Qobg4iUbu7w3HPhTqwDB8IFF8D06aGthlnS0YlISUsr2TCz582sjpltDUwGpprZZWmu4wJgWsrw7cA97t4cWAr0K0zAIlK6ffstHHYYnHgiNG0a7mVyzz1Qu3bSkYlIUtKt2Wjj7iuAnsA7QDPgpIJmMrOdgCOAx+OwAYcAr8ZJnorLFJEy7tdf4brroG3bkGD8+9/h7qzt2ycdmYgkLd0GolXNrCohMXjQ3TeYmacx373A34Hs/zT1gWXuvjEO/ww0zm1GM+sP9Ado0qRJmmGKSBLeey+0zZgxA44/Hu66C3bYIemoRKS0SLdmYyAwC9ga+NDMdgFW5DeDmR0JLHT3cUUJzN0fdfeO7t6xQYMGRVmEiGTY/PkhuejWLbTFeO+90FZDiYaIpEqrZsPd7wfuTyn60cy6FDDbgcBRZtYDqAHUAe4DtjGzKrF2YydgTuHDFpEkbdoUGn7+4x+wdm04fXL55VCjRtKRiUhplG4D0fpmdr+ZjTezcWZ2H1A3v3nc/Up338ndmwJ9gJHufgIwCjguTtYXGFL08EWkpI0fHy5lPecc+MMfYNIkuPZaJRoikrd0T6O8CCwCjiUkCouAl4q4zsuBi81sBqENxxNFXI6IlKAVK+DCC0OCMXs2PP88DB8OLVokHZmIlHbpNhBt5O43pgzfZGZ/TXcl7j4aGB1ffw90SndeEUmWO/znP6GvjHnzwq3gb74Zttkm6chEpKxIt2ZjuJn1iR16VTKz3sB/MxmYiCTvhx/CDdN69YLtt4cxY+Chh5RoiEjhpJtsnAE8D6yPjxeBM81spZnle1WKiJQ969eHG6W1aQMffhg65fryS+ikOkkRKYJ0r0ZR338iFcSHH8JZZ8G0aaF78XvvhZ12SjoqESnL0r7rq5kdBWTfDHq0u7+VmZBEJAmLFsHf/w6DB4duxt96C444IumoRKQ8SPfS19sI9ziZGh8XmNmtmQxMREpGVhY88QS0agXPPgtXXglTpijREJHik27NRg9gb3fPAjCzp4AJwJWZCkxEMm/y5HDK5JNP4KCD4OGHYY89ko5KRMqbwtxiPrX9eb4deolI6bZ6dejxs337cOv3J5+EDz5QoiEimZFuzcatwAQzGwUYoe3GFRmLSkQy5s034dxzQ8dc/frB7bdD/fpJRyUi5Vm6V6O8YGajgT/EosvdfX7GohKRYvfTT6FjrtdfDzUYH30EnTsnHZWIVATpNhA1oCuh3cZQoJqZ6Yp7kTJg40a4+25o3RrefTfUZEyYoERDREpOum02/g3sD/wtDq8EHspIRCJSbMaMgY4d4ZJLoEsXmDo1XN5atWrSkYlIRZJusrGvu58D/Arg7kuBahmLSkS2yNKl4SqTAw6AJUvgtddg6NDQf4aISElLN9nYYGaVAQcwswZAVsaiEpEicQ99ZbRsCY8/DhddFGozjjkGzJKOTkQqqnSvRrkfeB3Y3sxuJtxm/p8Zi0pECu2bb8IdWUeNgn33Dbd/33vvpKMSEUn/apTnzGwcoZGoAT3dfVpGIxORtKxdG26advvtULMmPPIInHEGVCpMLzoiIhmUVrJhZv3c/QlgekrZbe6uvjZEEjR8OAwYADNnwoknwr/+BQ0bJh2ViMjm0v3vc6yZnZA9YGYPAQ0yE5KIFGTePOjTB/70J6hcGUaMgGeeUaIhIqVTum02jgWGmlkWcDiwzN37ZS4sEcnNpk3h/iVXXQXr1sENN4RLWatXTzoyEZG85ZtsmFm9lMHTgTeAT4Drzayeu/+SyeBE5HfjxoXLWceOhcMOg3//G5o3TzoqEZGCFVSzMY5wuaulPB8RHw7smtHoRIQVK+Cf/4QHH4Ttt4cXX4TevXUpq4iUHfkmG+7erKQCEZHNucOrr4b7mcyfHxqC3nQTbLNNwfOKiJQm6bbZEJESNHNmuDPru+/CPvvAkCHwhz8UPJ+ISGmkK/FFSpF16+Dmm2HPPeGTT+C+++CLL5RoiEjZVlAD0aruvqGkghGpyEaPDj2ATp8OvXrBPfdA48ZJRyUisuUKqtn4zMzeMLOzzKxpCcQjUuEsWgR9+4a7sq5bB8OGwcsvK9EQkfIj32TD3TsCF8bBe83sSzO7x8y6mZmu7BfZAllZ4WZpLVvCCy+EvjMmT4bu3ZOOTESkeBXYZsPdZ7n7I+7eEzgAeBM4FPjIzN7OdIAi5dHXX0PnzuEeJnvtBRMnhitNatZMOjIRkeJXqKtRYvuNkfGBmamiV6QQVq+G664L7TG23RaeegpOOkl9ZohI+bZFl766+5ziCkSkvBs6FM47D2bPDjUat94K9esnHZWISOapnw2RDJs9G84/P/SV0bYtfPwxHHhg0lGJiJQc9bMhkiEbNoRbvrduDe+9B3fcEe5vokRDRCqatGo2zKwFcBmwS+o87n5IhuISKdM+/TTcNG3SJDjqKLj/fthll6SjEhFJRrqnUV4BHgEeAzZlLhyRsu2XX+CKK+Cxx2DnneGNN+Doo5OOSkQkWekmGxvd/eGMRiJShrnDM8/ApZeGhOPSS+Haa6FWraQjExFJXkHdldeLL980swHA68C67PHu/ksGYxMpE6ZPD92Mjx4N++8PjzwS+s4QEZGgoJqNcYAD2b0AXJYyzoFdMxGUSFmwdi3ccgvcfnuowXj0UejXDyqp2bWIyGbyTTbcvVlJBSJSlrz7LpxzDnz/PZx8Mtx5J2y/fdJRiYiUTmn9BzOzc8xsm5ThbeNpFZEKZe5c+Otfw/1LqlaFkSNDL6BKNERE8pZuhe8Z7r4se8DdlwJnFDSTmdUwsy/MbKKZTTGz62N5MzP73MxmmNlLZlataOGLlIxNm+CBB6BVq9AT6I03hvuZdOmSdGQiIqVfuslGZbPf795gZpWBdBKEdcAh7t4O2Bs43Mz2A24H7nH35sBSoF/hwhYpOWPHwr77hl5ADzgg3Jn16quhuu57LCKSlnSTjf8CL5lZVzPrCrwAvFvQTB6sioNV48OBQ4BXY/lTQM9CRS1SApYvD/cy6dQpnD556SV45x3YbbekIxMRKVvS7Wfj70B/4Ow4/B7weDozxlqQcUBz4CFgJrDM3TfGSX4G/ufusWbWP66TJk2apBmmyJZzh5dfhgsvhIUL4dxzw2mTunWTjkxEpGwqMNmIycLT7n4CoRfRQnH3TcDesYHp60CrNOd7FHgUoGPHjl7Y9YoUxYwZ4SqT4cOhQwd4803o2DHpqEREyrYCT6PEZGGXLW3EGRuYjgL2B7Yxs+xEZydAt6qXRK1bF2ov9twTPvssNAb9/HMlGiIixSHd0yjfA5+Y2VBgdXahu9+d30xm1gDY4O7LzGwr4DBC49BRwHHAi0BfYEgRYhcpFqNGhR5Av/kmXNZ6992w445JRyUiUn6km2zMjI9KQO1CLL8R8FQ8FVMJeNnd3zKzqcCLZnYTMAF4ohDLFCkWCxeGe5g88wzsumvoqOtPf0o6KhGR8ietZMPds/vHqBWHV+U/x2/zfQ20z6X8e6BT+mGKFJ+sLHj8cbj8cli9Gv75T7jySthqq6QjExEpn9JKNsxsT+AZoF4cXgyc7O5TMhibSLGbOBHOOgvGjAkdcv3736GjLhERyZx0+9l4FLjY3Xdx912AS4DHMheWSPFatSqcMunQAWbODKdORoxQoiEiUhLSbbOxtdqvMS4AAB4vSURBVLuPyh5w99FmtnWGYhIpVm+8ETrn+vln6N8fbr0V6tVLOioRkYoj3ZqN783sn2bWND6uJlyhIlJq/fgjHHUUHHNMSC4+/RQGDlSiISJS0tJNNk4DGgCvAf8BtgNOzVRQIltiwwa44w5o0ybclfVf/4Jx42D//ZOOTESkYkr3NMqh7n5+aoGZ9QJeKf6QRIruk09CA9DJk6FnT7jvPlBv9yIiyUq3ZuPKNMtEErFkCZxxBnTuDCtWwJAh8PrrSjREREqDfGs2zKw70ANobGb3p4yqA2zMfS6RkuMOTz8drjRZtgz+/ne45hrYWs2XRURKjYJOo8wFxgJHEe7cmm0lcFGmghJJx7RpoZvxDz6AAw+Ehx+Gtm2TjkpERHLKN9lw94nARDNbCrzl7lklE5ZI3tasgZtvhjvvhNq1Q2+gp54KldI9KSgiIiUq3cNzb+A7M7vDzNQNkiTmnXfCnVlvuQVOOAGmT4d+/ZRoiIiUZmkdot39RMI9TmYCg83sMzPrb2aFuSmbSJHNmQO9ekGPHlCjBoweDU8+CQ0aJB2ZiIgUJO3/g+6+AniVcFv4RsAxwHgzOy9DsYngHroWb9MG3nornD756iv44x+TjkxERNKV7o3YjiJ04tUceBro5O4LzawmMBV4IHMhSkX1yy+hAejLL8NBB8HgweFW8CIiUrak26nXscA97v5haqG7rzGzfsUfllR0I0ZA376wcCHcdlu4tLVy5aSjEhGRoki3zUZf4FszO8rM/mxmO6SMG5Gx6KTC+fVXuOQSOPTQcKXJmDFw+eVKNEREyrK0ko1Ye/EF8BfgOGCMmZ2WycCk4pk0CTp1grvvhnPPDfcz2WefpKMSEZEtle5plL8D7d19CYCZ1Qc+BQZlKjCpOLKy4N574corwx1Zhw2D7t2TjkpERIpLusnGEkKvodlWxjKRLfLzz6FtxsiR4cZpjz6qy1lFRMqbdJONGcDnZjYEcOBo4GszuxjA3e/OUHxSjr38Mpx5Zrgl/OOPw2mngVnSUYmISHFLN9mYGR/ZhsRndeolhbZ8OZx3Xug/Y9994dlnoXnzpKMSEZFMSSvZcPfrMx2IVAwffQQnnRROn1x3HVx1FVRJN+UVEZEySXeUkBKxfj384x+h58+qVeGTT+Daa5VoiIhUBDrUS8ZNmwYnngjjx8MZZ4RLW2vVSjoqEREpKarZkIxxh4ceCn1lzJ4Nb7wRrjZRoiEiUrGk26lXCzMbYWaT4/BeZnZ1ZkOTsmz+fDjiiNA5V5cuocOuo49OOioREUlCujUbjwFXAhsA3P1roE+mgpKybcgQaNsWRo0KNRtvvw077FDwfCIiUj6lm2zUdPcvcpRtLO5gpGxbtSq0yejZE5o0CW00BgxQ3xkiIhVdusnGYjPbjdChF2Z2HDAvY1FJmTNmDOy9NzzxROh2/LPPoHXrpKMSEZHSIN2rUc4BHgVamdkc4AfgxIxFJWXGxo1w881w442w007wwQdw0EFJRyUiIqVJup16fQ8camZbA5XcfWVB80j5N2NGuKT1889DR10PPAB16yYdlYiIlDZpJRtmtg1wMtAUqGLxJLy7n5+xyKTUcg+nSy68EKpVg5degt69k45KRERKq3RPowwDxgCTgKzMhSOl3aJFoRHokCHQtSsMHhxOn4iIiOQl3WSjhrtfnNFIpNR75x049VRYujT0AnrBBVBJ3cKJiEgB0v2peMbMzjCzRmZWL/uR0cik1FizJnTO1aMHbL89fPklXHSREg0REUlPujUb64E7gauIl7/G510zEZSUHuPHwwknwPTpcPHF4cqTGjWSjkpERMqSdJONS4Dm7r44k8FI6bFpE9x5J/zzn9CwIbz/fmijISIiUljpVoTPANYUZsFmtrOZjTKzqWY2xcwuiOX1zOw9M/suPm9b2KAls2bNCvczufJKOOYY+PprJRoiIlJ06dZsrAa+MrNRwLrswgIufd0IXOLu482sNjDOzN4DTgFGuPttZnYFcAVweZGil2LlDs89B+ecE14//XToR0PdjYuIyJZIN9l4Iz7S5u7ziF2au/tKM5sGNAaOBg6Okz0FjEbJRuKWLoWzzw59Zhx0UEg0mjZNOioRESkP0u1B9KktWYmZNQXaA58DDWMiAjAfaLgly5YtN2IE9O0LCxbArbfCZZdB5cpJRyUiIuVFvsmGmb3s7r3NbBK/X4XyG3ffq6AVmFkt4D/Ahe6+wlLq5N3dzex/lhvn6w/0B2jSpElBq5EiWLcOrroK7roLWrYMHXV16JB0VCIiUt4UVLNxQXw+sigLN7OqhETjOXd/LRYvMLNG7j7PzBoBC3Ob190fJdz8jY4dO+aakEjRTZoULmmdNCm00bjjDqhZM+moRESkPMr3apSU0x0D3P3H1AcwIL95LVRhPAFMc/e7U0YNBfrG132BIUULXYoiKwvuuQf+8AdYuBDefhsefFCJhoiIZE66l74elktZ9wLmORA4CTjEzL6Kjx7AbcBhZvYdcGgclhLw88/QrVvonOvww0OtRo8eSUclIiLlXUFtNs4m1GDsamZfp4yqDXyS37zu/jGQ10WT6rWhhL3yCpx5Zmin8dhj0K+fLmkVEZGSUVCbjeeBd4BbCf1hZFvp7r9kLCopNitWwHnnhUtZ990XnnkGdt896ahERKQiyTfZcPflwHLgbyUTjhSnjz6Ck0+Gn36Ca6+Fq6+GKun2rCIiIlJMdN/Ocmj9+nBJ68EHh/4yPvoIrrtOiYaIiCRDPz/lzPTpoYvxceNCu4x77oHatZOOSkREKjLVbJQT7vDww7DPPuFGaq+9Bo8/rkRDRESSp5qNcmD+/FCLMWxYuKR10CBo1CjpqERERALVbJRxQ4ZA27YwcmTonGvYMCUaIiJSuijZKKNWrYL+/aFnT9h5Zxg/PnQ7rr4zRESktFGyUQZ9/jm0bx/aZFxxBYwZA61bJx2ViIhI7pRslCEbN8INN8CBB4bLW0ePDreEr1Yt6chERETypgaiZcTMmeGS1jFjwvODD0LduklHJSIiUjDVbJRy7uHqknbtQh8aL7wQuhxXoiEiImWFko1SbPFi+MtfwmWtnTrB119Dnz5JRyUiIlI4SjZKqXffDZe0DhsG//oXvP9+uOpERESkrFGyUcqsXRvu0tq9O2y3HXz5JVxyCVTSJyUiImWUfsJKkQkToEOH0PjzootCorHXXklHJSIismWUbJQCmzbB7bfDvvvC8uXw3ntw991Qo0bSkYmIiGw5XfqasB9/hJNPhg8/hOOOg4EDoV69pKMSESk+48aN275KlSqPA3uiP7nlVRYweePGjad36NBhYc6RSjYS4g7PPw8DBoTXTz0FJ52k7sZFpPypUqXK4zvssEPrBg0aLK1UqZInHY8Uv6ysLFu0aFGb+fPnPw4clXO8MswELF0Kxx8fOudq2xYmTgy1G0o0RKSc2rNBgwYrlGiUX5UqVfIGDRosJ9Re/e/4Eo6nwhs5MjT6fPVVuPlm+OADaNYs6ahERDKqkhKN8i9+xrnmFUo2Ssi6dXDppdC1K2y9NXz2GfzjH1C5ctKRiYiIZJaSjRIweXLoAfSuu0IbjfHjoWPHpKMSEZG8XHHFFTsU17IuvvjiHa+55pqGeY0//vjjmwwfPnzrTp06tfzwww9rFtd6i9PDDz9cr0WLFm1atGjRpn379q0+++yzrQozv5KNDMrKgnvvDYnF/Pnw1lvw0ENQs1TuSiIiku3+++9vVJjps7Ky2LRpU5HWNX78+FqHHHLI6iLNXEKaN2++7pNPPvnm22+/nXrllVfOPfPMM3cpzPxKNjJkzhz4059C51zdusGkSXDEEUlHJSJS8XzzzTfVmjVrtsdRRx3VbNddd93j8MMP33XlypWVhg4dWvvQQw/dLXu6119/vc5hhx2224ABAxqvW7euUqtWrdocddRRzQCuu+66hrvvvvseu++++x433HDD9tnLbdq06Z7HHHNM0xYtWuwxc+bMaq+++mqdNm3atG7ZsmWb/fffv0X2sqdNm7ZVp06dWu60005tb7rppu2zy8ePH19j1113/bVKld8vDt20aRPHHnts0/PPP39HgNdee63O3nvv3apNmzatu3fvvuvy5csrAQwZMqR269at27Ro0aJNr169mq5du9YAGjdu3Pass87aqUWLFm3atm3bevLkydUBBg0atO3uu+++R8uWLdt07NixZWG24WGHHba6QYMGmwC6dOmyev78+dUKM78ufc2AV1+F/v1DO42BA+GMM3SliYgIwGmnsfPkyRRr/e6ee7Jm0CB+ym+aWbNm1Rg4cOCsbt26re7Vq1fTO++8s8F111234IILLmgyd+7cKjvuuOPGQYMG1T/11FMXH3/88csHDx68/fTp06cCfPTRRzWff/75+uPGjZvm7nTo0KF1165dV2633XabZs+eXf2JJ574oWvXrrPmzp1b5dxzz206evTo6a1atVq/YMGC31rlzZgxo8ann376zbJlyyq3bt16z8suu2xR9erVfejQoXW7deu2PHu6DRs2WM+ePZu1adNm7e233z5/3rx5VW655ZZGH3744bd16tTJuuqqq3a48cYbG95www3zzzzzzGbDhw//Zq+99lp3zDHHNL3zzjsbXHPNNQsB6tatu/Hbb7+d+uCDD9Y/77zzdh41atSM2267rdHw4cO/bdas2YbFixcXucXgAw88sF2XLl2WFzzl71SzUYxWrIBTToFevWD33UP34/37K9EQEUnaDjvssL5bt26rAU466aQln376aa1KlSrRu3fvJY899li9xYsXVx4/fnytXr16/c+P6OjRo2v16NFjWZ06dbLq1q2bdcQRRywdNWpUbYBGjRqt79q16+o43dadOnVa2apVq/UADRs2/O28Srdu3ZZttdVW3qhRo4316tXb8PPPP1cBeP/99+v07NlzRfZ0AwYM2CU70che5syZM2t06tSpVatWrdq8+OKL9WfPnl1t4sSJNXbaaad1e+211zqAU045ZcnHH39cO3s5ffv2/QXgjDPO+GXChAm1ADp27LjqhBNOaHrXXXdtt3HjxiJtxzfffLP2s88+u9199933c2HmU81GMfn449Ap1+zZcM01cPXVULVq0lGJiJQuBdVAZIrl+NeXPXz22WcvOeKII5rXqFHD//znPy+tWsgDd82aNbPSma569eq/XfpbuXJlNm7caCtXrqy0YsWKyk2bNt2QPa5jx46rPvroozpr1qxZULNmTXd3OnfuvOLNN9/8IXV5BTXQrJRy904zc4Dnn39+9siRI7ceOnRo3Q4dOrQZN27c1B122OG3hOi8885r/N5779UFyK7VSfX5559vNWDAgF3efvvt71LnS4dqNrbQ+vVw1VXwxz+GO7N+/DFcf70SDRGR0mTevHnV3n///a0BnnvuuXoHHHDAKoCmTZtuaNiw4Ya77rqrUf/+/RdnT1+lShVft26dAXTp0mXVsGHDtonJQaVhw4Zt26VLl5U513HwwQev/uKLL2pPnz69GkDqaZTcvP3227U7d+682XLOPPPMxd26dVt+5JFH7rZhwwYOPvjg1WPHjq2V3e5ixYoVlb7++uvq7dq1+3XOnDnVssuffvrp+gcddNBvy3r66afrATzxxBPbtm/ffjXAlClTqh9yyCGr77333rnbbrvtxu+//36zdhcPPPDAnOnTp0/NLdH47rvvqvXq1Wu3QYMG/ZBdm1IYqtnYAtOnh15Ax42D004LV57Url3wfCIiUrKaNm366wMPPLB9//79a+6+++6/XnrppYuyx/Xp02fJQw89VGWfffb5NbvshBNOWNS6des2e+6555qhQ4f+cPzxxy/ZZ599WgOcdNJJiw488MC133zzzWY/1jvuuOPG+++/f9YxxxzTPCsri/r162/49NNPv8srpmHDhtXt3bv30pzl11133YKLLrqo8l/+8pdmb7zxxg8DBw6c1adPn13Xr19vANdee+2cvfbaa90jjzwyq1evXrtt2rSJdu3arUl9T0uXLq3cokWLNtWqVfMXX3zxe4CLLrpop1mzZlV3d+vcufOK/fbbb2262+/qq69utGzZsirnnXfeLhCSscmTJ09Ld35zL/2dunXs2NHHjh2bdBi/cYdHHoFLLgmXsT76KPzlL0lHJSKyOTMb5+6J9+ozceLEWe3atVtc8JSZ8c0331Q78sgjd//uu++m5Db+5JNPbtK+ffs1F110UYnG2KZNm9YTJkyYnnqKpTg0bty47dixY6c1atSoaA0ztsDEiRO3a9euXdOc5arZKKQFC6BfP3j77XBp65NPQqNCXY0tIiKlxR577NF6q622yho4cGCJtyWZOnVq2jUDZZ2SjUIYOhROPx1WroT774dzz9WVJiIipV3Lli3X51WrMWXKlHL3gz9nzpxJSceQkxqIpmH1ajjzTDj6aGjcGMaOhfPOU6IhIpKmrKysLB0xy7n4Ged6dY6SjQJ88QW0bw+PPQaXXw5jxsAeeyQdlYhImTJ50aJFdZVwlF9ZWVm2aNGiusDk3MbrNEoeNm6EW28Nl7E2bgyjRoXLW0VEpHA2btx4+vz58x+fP3/+nuhPbnmVBUzeuHHj6bmNVLKRi5kzQwddn30GJ5wADz4I22yTdFQiImVThw4dFgJHJR2HJEcZZgp3GDQI9t4bpk6F55+HZ59VoiEiIrIlMppsmNkgM1toZpNTyuqZ2Xtm9l183jaTMaRr8WI47rhwWWvHjvD11/C3vyUdlYiISNmX6ZqNwcDhOcquAEa4++7AiDicqP/+F/baC958E+68E0aMgCZNko5KRESkfMhosuHuHwK/5Cg+Gngqvn4K6JnJGPKzdi2cfz4cfjjUqwdffgmXXhrucSIiIiLFI4mf1YbuPi++ng80zG0iM+tvZmPNbOyiRYtym2SLTJgAHTrAAw/ABReERKNdu2JfjYiISIWX6H94DzdmybVPeHd/1N07unvHBg0aFNs6N22CO+6AffeF5cth+PBwA7Wt8r1Zr4iIiBRVEpe+LjCzRu4+z8waAQtLasWzZ8PJJ8MHH8Cxx8LAgVC/fkmtXUREpGJKomZjKNA3vu4LDCmJlT7/fGgEOm4cDB4Mr7yiRENERKQkZPrS1xeAz4CWZvazmfUDbgMOM7PvgEPjcMYsXQrHHx8659pjD5g4Efr21X1NRERESkpGT6O4e149VXTN5HqzTZoERxwB8+bBTTeFe5tUUZ+pIiIiJapc//Q2aQKtW8N//gN/+EPS0YiIiFRM5TrZqFs3dNglIiIiyVH3VSIiIpJRSjZEREQko5RsiIiISEYp2RAREZGMUrIhIiIiGaVkQ0RERDJKyYaIiIhklJINERERySgLd3kv3cxsEfDjFixiO2BxMYVTnBRX4SiuwlFchVMe49rF3RsUZzAiRVEmko0tZWZj3b1j0nHkpLgKR3EVjuIqHMUlkjk6jSIiIiIZpWRDREREMqqiJBuPJh1AHhRX4SiuwlFchaO4RDKkQrTZEBERkeRUlJoNERERSYiSDREREcmoMplsmNnOZjbKzKaa2RQzuyCW1zOz98zsu/i8bSw3M7vfzGaY2ddmtk/KsvrG6b8zs75bGFcNM/vCzCbGuK6P5c3M7PO4/pfMrFosrx6HZ8TxTVOWdWUs/8bM/rQlcaUss7KZTTCzt0pLXGY2y8wmmdlXZjY2liX6OcblbWNmr5rZdDObZmb7Jx2XmbWM2yn7scLMLkw6rri8i+I+P9nMXojfhdKwf10QY5piZhfGshLfXmY2yMwWmtnklLJii8PMOsTv0Yw4rxVti4lkiLuXuQfQCNgnvq4NfAu0Ae4ArojlVwC3x9c9gHcAA/YDPo/l9YDv4/O28fW2WxCXAbXi66rA53F9LwN9YvkjwNnx9QDgkfi6D/BSfN0GmAhUB5oBM4HKxbDdLgaeB96Kw4nHBcwCtstRlujnGJf5FHB6fF0N2KY0xJUSX2VgPrBL0nEBjYEfgK1S9qtTkt6/gD2ByUBNoArwPtA8ie0F/B+wDzA5E/s58EWc1uK83YtjP9NDj+J6JB5AsbwJGAIcBnwDNIpljYBv4uuBwN9Spv8mjv8bMDClfLPptjCmmsB4YF9C739VYvn+wH/j6/8C+8fXVeJ0BlwJXJmyrN+m24J4dgJGAIcAb8X1lIa4ZvG/yUainyNQl/DjaaUprhyxdAM+KQ1xEZKNnwg/glXi/vWnpPcvoBfwRMrwP4G/J7W9gKZsnmwUSxxx3PSU8s2m00OP0vAok6dRUsUq2PaEWoSG7j4vjpoPNIyvsw+G2X6OZXmVb0k8lc3sK2Ah8B7h39kyd9+Yyzp+W38cvxyon4m4gHsJB9qsOFy/lMTlwHAzG2dm/WNZ0p9jM2AR8KSF006Pm9nWpSCuVH2AF+LrRONy9znAv4DZwDzC/jKO5PevycBBZlbfzGoSagx2pvR8jsUVR+P4urjjEyk2ZTrZMLNawH+AC919Reo4d3fCD1mJcvdN7r43oSahE9CqpGPIycyOBBa6+7ikY8lFZ3ffB+gOnGNm/5c6MqHPsQqhyvthd28PrCZUcycdFwCx7cNRwCs5xyURV2xrcDQhSdsR2Bo4vCRjyI27TwNuB4YD7wJfAZtyTJPY51ga4xDJlDKbbJhZVUKi8Zy7vxaLF5hZozi+EaF2AWAO4R9Ntp1iWV7lW8zdlwGjCNXH25hZlVzW8dv64/i6wJIMxHUgcJSZzQJeJJxKua8UxJX9rxh3Xwi8TkjQkv4cfwZ+dvfP4/CrhOQj6biydQfGu/uCOJx0XIcCP7j7InffALxG2OdKw/71hLt3cPf/A5YS2nclvb2yFVccc+Lr4o5PpNiUyWQjtrR+Apjm7nenjBoKZLfQ7ktoy5FdfnJs5b0fsDxWX/4X6GZm28Z/Z91iWVHjamBm28TXWxHakUwjJB3H5RFXdrzHASPjP5yhQJ/Yar8ZsDuhAViRuPuV7r6TuzclVL+PdPcTko7LzLY2s9rZrwnbfzIJf47uPh/4ycxaxqKuwNSk40rxN34/hZK9/iTjmg3sZ2Y143cze3slun8BmNn28bkJ8BdCA+mkt1e2YokjjlthZvvF7X9yyrJESoekG40U5QF0JlQ5fk2oGv2KcD62PqER5HeEluf14vQGPERoPzEJ6JiyrNOAGfFx6hbGtRcwIcY1Gbgmlu9KOGjOIFR9V4/lNeLwjDh+15RlXRXj/YZibFkOHMzvV6MkGldc/8T4mAJcFcsT/Rzj8vYGxsbP8g1C6//SENfWhFqAuillpSGu64Hpcb9/hnBFSeL7PfARIfGZCHRNansRksN5wAZCzVm/4owD6Bi3/UzgQXI0btZDj6Qf6q5cREREMqpMnkYRERGRskPJhoiIiGSUkg0RERHJKCUbIiIiklFKNkRERCSjlGxIuWZmmyzcIXWimY03swMKmH4bMxuQxnJHm1nHIsY0LLs/FhGRikDJhpR3a919b3dvR7jR160FTL8N4a6kGePuPTz0MCsiUiEo2ZCKpA6hy2rMrJaZjYi1HZPM7Og4zW3AbrE25M447eVxmolmdlvK8nqZ2Rdm9q2ZHZRzZWbWyMw+jMuanD2Nmc0ys+3M7Kw47isz+8HMRsXx3czssxjbKxbuASQiUmapUy8p18xsE6EXxhqEW3Ef4u7j4j05arr7CjPbDhhD6B57F0IPq3vG+bsTbk1+qLuvMbN67v6LmY0Gxrn7JWbWA7jY3Q/Nse5LgBrufrOZVY7rW2nhHjUd3X1xnK4qMBK4A/iMcG+R7u6+2swuJ/S8eUMmt5OISCZVKXgSkTJtrYe78GJm+wNPm9mehC6hb7Fwl9kswi25G+Yy/6HAk+6+BsDdf0kZl30DwHFA01zm/RIYFJOJN9z9qzxivI9wf5A3Ldyhtw3wSbjNBdUICYiISJmlZEMqDHf/LNZiNCDcS6cB0MHdN8TahhqFXOS6+LyJXL5L7v5hTGaOAAab2d3u/nTqNGZ2CqE25dzsIuA9d/9bIWMRESm11GZDKgwzawVUJt7IDFgYE40uhB98gJVA7ZTZ3gNONbOacRn1CrG+XYAF7v4Y8DjhNvWp4zsAlwInuntWLB4DHGhmzeM0W5tZi8K9UxGR0kU1G1LebWVm2acvDOjr7pvM7DngTTObRLi763QAd19iZp+Y2WTgHXe/zMz2Bsaa2XpgGPCPNNd9MHCZmW0AVhFu/Z3qXKAeMCqeMhnr7qfH2o4XzKx6nO5q4NtCv3MRkVJCDURFREQko3QaRURERDJKyYaIiIhklJINERERySglGyIiIpJRSjZEREQko5RsiIiISEYp2RAREZGM+n+VeVGWVKXj8QAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] @@ -554,29 +495,130 @@ } ], "source": [ - "plot_relative_time(results, [2], max_batch_size=max_batch_size)" + "plot_relative_time(experiments_neq, results, [2], max_batch_size=max_batch_size)" ] }, { "cell_type": "markdown", - "id": "f7dc206c", + "metadata": {}, + "source": [ + "### Learned kernel MMD detector\n", + "\n", + "We conduct similar experiments as for the MMD detector for $N_\\text{ref} = N_\\text{test}$ and `n_features=50`. We use a deep learned kernel with an MLP followed by Gaussian RBF kernels and project the input features on a `d_out=2`-dimensional space. Since the learned kernel detector computes the kernel matrix in a batch-wise manner, we can also scale up the number of instances for the PyTorch backend without running out-of-memory." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "experiments_eq = {\n", + " 'keops': {\n", + " 0: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 3, 'n_features': 50},\n", + " 1: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 3, 'n_features': 50},\n", + " 2: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 3, 'n_features': 50},\n", + " 3: {'n_ref': 20000, 'n_test': 20000, 'n_runs': 3, 'n_features': 50},\n", + " 4: {'n_ref': 50000, 'n_test': 50000, 'n_runs': 3, 'n_features': 50},\n", + " 5: {'n_ref': 100000, 'n_test': 100000, 'n_runs': 3, 'n_features': 50}\n", + " },\n", + " 'pytorch': {\n", + " 0: {'n_ref': 2000, 'n_test': 2000, 'n_runs': 3, 'n_features': 50},\n", + " 1: {'n_ref': 5000, 'n_test': 5000, 'n_runs': 3, 'n_features': 50},\n", + " 2: {'n_ref': 10000, 'n_test': 10000, 'n_runs': 3, 'n_features': 50},\n", + " 3: {'n_ref': 20000, 'n_test': 20000, 'n_runs': 3, 'n_features': 50},\n", + " 4: {'n_ref': 50000, 'n_test': 50000, 'n_runs': 3, 'n_features': 50},\n", + " 5: {'n_ref': 100000, 'n_test': 100000, 'n_runs': 3, 'n_features': 50}\n", + " }\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 13, "metadata": { - "pycharm": { - "name": "#%% md\n" - } + "scrolled": true }, + "outputs": [], + "source": [ + "results = {backend: {} for backend in backends}\n", + "\n", + "for backend in backends:\n", + " exps = experiments_eq[backend]\n", + " for i, exp in exps.items():\n", + " results[backend][i] = experiment(\n", + " 'learned_kernel', backend, exp['n_runs'], exp['n_ref'], exp['n_test'], exp['n_features']\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We again plot the absolute and relative (PyTorch / KeOps) mean prediction times for the learned kernel MMD drift detector for different feature dimensions:" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgcAAAEWCAYAAADywzSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXxU1fnH8c8Twr4jiCxCQNlxBalotShIUVS0Vkq1iLtWbV1b6eJaq/6q2Lr83EWlihvVioh1xe2HikDZBBREkCVA2PclyfP749zoJCZkIZM7Sb7v12tec+fc7bkzk9xnzjn3XHN3RERERPKkxR2AiIiIpBYlByIiIpKPkgMRERHJR8mBiIiI5KPkQERERPJRciAiIiL5KDmoAszMzezAPcx/2MxuSHj9azNbZWZbzGyfMuwvI9pnelljLi/RMXSsgP20i/ZVowL21dLMPjSzzWY2Ktn7k5Izsz+a2eNxxyGSbEoOSsHMFpvZLjNrXqD8v9HJMiN6/VT0ekiB5f4elZ8bvT7XzHKik84WM/vGzJ40s87lGbe7X+ruf4n2WRO4Bxjo7g3cfW1xycXeiI7x43La1vtmdmFiWXQMi8pj+wX2tdjMBiTs59toXznlva9CXAysARq5+7V7uzEza2Vm481sReL3NGF+bTMbbWabzGylmV1TYH5/M5tvZtvMbJKZtd/bmOJgZjeb2TOlWL6fmS1LLHP32939wqLWEakqlByU3jfAL/NemNlBQL1ClvsKOCdhuXRgKPB1geU+cfcGQGNgALAdmGZmPcsj2EJ+6bYE6gBflMf2JSnaA3O9DCOUFVGbkwv8BzijiNVuBjpF+z0O+L2ZDYq21xx4GbgBaAZMBV4obVyllQq1UiLVmrvrUcIHsBj4M/B5QtndwJ8ABzKisqei8lVA06jsZOAN4GPg3KjsXODjQvYzARi3hzh+B2QCK4Dzo30fmLDvh4CJwFZCwvEUcBvQOSpzYAvwHvBh9HprVPaLQvZXIzqeNcAi4PJonfRofmPgiSim5dG+agDdgB1ATrTtDdHytaPtfRu9Rw8DdRP2NwSYAWwiJFODgL9G29kRbeuBaNnEY28MjAGygCXRZ5WW+F5H+11PSPJOLOL9/SfhhLo92tfvgYwCx/x+dJyTo2VeA/YBno3i/jzv+xAt3xV4G1gHfAkMLWLfTwG7gV3RdgdE79c/os97RTRdO1q+H7AMuB5YCfxzD9+bdBK+pwnlKwg1SXmv/wI8H01fDExOmFc/el+67uFv5A/A3Oh9fhKokzD/5Oiz3RC9dwcXWPd6YBawM4p3MeH7PovwHX2CkOC+AWwG3uH7v7F+wLJC4hlA+A7tit7bLcDMaP55wLxoW4uASwocZ260/BagNSGReiZh+6cSEu0N0XeiW4F9XxfFvpGQVNWJ5jUn/J1viL4THxF9V/XQIxUesQdQmR4J/2i+JJz4akT/mNvzw+TgNuBR4NdR2YuEGoeSJAfnA6uKiGEQ4YTaM/oHNpYfJgcbgaMJNUN18uKJ5meQcJKLyr5bv4h9XgrMB/Yn/HqcRP4T5SvAI1E8+wJTEv7J/uAYgb8D46NtNSScWO+I5vWJ4j8hir8N0Yko+ud7YYFtJR77GODVaJsZhNqbCxLi2A1cFH1uvyacFG1Pn3XC63zvWxTLQuAAQlIyN9rfAMJJbQzwZLRsfWAp4USUDhxGSLS6F7Hv7z6v6PWtwKfRe9uCcFL9SzSvH5AN/A8hiahb2DajZX+QHABNo7KWCWU/B2ZH0/cCDxXYzhzgjD28b3MSviv/x/ffvcOA1cCPos9gRLR87YR1Z0Tr1k0o+5SQELSJ1p8ebasOIcG9KeG9KDQ5iKZvJuHEHpUNjj5DA34CbAMO38P2vtsG3yfbJwA1CUnkQqBWwr6nEJKKZoQk5NJo3h2EpLhm9DiGIr6LeugRx0PNCmXzT0KTwQmEP/jlRSw3BjjHzJoQ/vH8u4TbX0H4Z1KYoYSTzhx330r4Z1XQq+7+f+6e6+47SrjPPRkK/MPdl7r7OsI/NiB0ngNOAq5y963uvppw8h9W2IbMzAi/Rq9293Xuvhm4PWH5C4DR7v52FP9yd59fXIBR88kw4A/uvtndFwOjgOEJiy1x98c89Bt4GmhFOOmU1ZPu/rW7byT8kv3a3d9x92zgJcIJDMKv5cXu/qS7Z7v7f4F/AWeWcD9nA7e6+2p3zwJuKXBcuYQT5E53317KY2gQPW9MKNtISLDy5m8kv8T5hXkg4bvyV75vhrsYeMTdP3P3HHd/mlBDcGTCuvdF6yYex/3uvsrdlxN+YX/m7v+Nvtuv8P37XGru/nr0Gbq7fwC8RThRl8QvgNej7+puQq1UXeCoAsezInovXgMOjcp3E75/7d19t7t/5O660Y2kDLXrlc0/CdXxHQgJQKHc/WMza0Fodpjg7tvDubFYbQhVjYVpDUxLeL2kkGWWlmQnpdC6wDYT99me8MsnM+HY0vYQQwtCH41pCcsb4ZckhF+NE8sQY/MojsTYlhDeyzwr8ybcfVu0/waU3aqE6e2FvM7bdnvgR2a2IWF+OuF7VBKt+eFxtU54nbUXSeCW6LkRockmb3pzwvxGBdZJnF+Ygt+VvFjbAyPM7DcJ82uR/1gK+96U9H0uNTM7EbiJUAuQRvhuzi7h6vk+F3fPNbOlFPGdI9RK5B3rXYTE/q3oe/iou99ZhkMQSQrVHJSBuy8htFmfROistSfPANeyhySiEKcTfiEVJpNwAs3TrrAQS7GvktjTPpcSfv01d/cm0aORu/coIpY1hH/oPRKWb+yhU2be9g4oIo49Hdcawq+xxJ707Si6Vqc45fkeLgU+SDjeJh6ufPh1CddfwQ+Pa0XC6zLH6u7rCZ/vIQnFh/B9h9UvEueZWX3C57OnDq0Fvyt5sS4F/lrgfajn7s8lhlS2IwFCFf93nYOj2qQWRW3bzGoTanDuJjSrNCEkplbY8oXI97lEtWL7U4LvXFS7da27dyT0W7jGzPoXt55IRVFyUHYXAMdHVft7ch+h+eHDPS1kZjXMrIOZ3U9o67yliEVfBM41s+5mVo/wq2dvrQL2NFbAi8BvzaytmTUFRubNcPdMQlXsKDNrZGZpZnaAmf0kYdttzaxWtHwu8BjwdzPbF8DM2pjZT6PlnwDOiy6fS4vmdS0uzqip4EXgr2bWMLrc7hpCclYWxb0npTEB6Gxmw82sZvQ4wsy6lXD954A/m1mL6OqBGynlcZlZHUKfBIDa0es8Y6LtN43e64sI/R4gVNv3NLMzonVuBGYV09RzefRdaUaoNcu7uuEx4FIz+5EF9c1ssJntqYmiNL4C6kTbrEnokFo7Yf4qIMPM8v7v1YrmZwHZUS3CwALL72NmjYvY34vA4Oi7WpPwI2AnoU/IHpnZyWZ2YJRQbCR0ts0t6YGKJJuSgzKK2imnlmC5de7+7h7aE/ua2RZCD/f3CVW2R7h7oVWb7v4Gobf6e4TOT++VJf4CbgaeNrMNZja0kPmPAW8CMwmdwQrWlpxD+Eeb10N9HKE9lSi+L4CVZrYmKrs+iv1TM9tE6HHeJTq+KYSOe38n/NP8gO9/nd0L/NzM1pvZfYXE+RvCr8dFhI6fY4HRJXsLfuAOwglzg5ldV8ZtAOFXIuGkM4zwa3Ml33cgLInbCJcQziJUeU+Pykoj78oLCJ1LE9v0byJcFbKE8H7f5e7/iWLPIlwC+VfCZ/sjiuhPkmAsIWFcFG33tmhbUwmJxwPRthYSOoqWi6jvx2XA44Rf71sJHYbzvBQ9rzWz6dHn8lvCSX49cBaho2ze9uYTErNF0fcgsfkDd/8S+BVwP6Hm6hTgFHffVYJwOxG+91uAT4AH3X1S6Y5YJHlMfWBEpLyY2WLCFSXvxB2LiJSdag5EREQkHyUHIiIiko+aFURERCQf1RyIiIhIPpV6EKTmzZt7RkZG3GGIiFQq06ZNW+PuLYpfUqqrSp0cZGRkMHVqsVcTiohIAjMrbGRVke+oWUFERETyUXIgIiIi+Sg5EBERkXyUHIiIiEg+Sg5EREQkHyUHIiIiko+SAxEREclHyYGISCXiDqtXxx2FVHVKDkREKpHRo+Gww2DlyrgjkaqsUo+QKCJS3Zx9NuzaBS1bxh2JVGWqORARSXHuMGoUrF8PderAr38NZnFHJVWZkgMRkRSWmwuXXw7XXQfPPBN3NFJdqFlBRCRF5ebCpZfCY4/B9dfDFVfEHZFUF6o5EBFJQbm5cNFFITH44x/hjjvUlCAVR8mBiEgKWrMG3n0XbrgBbrtNiYFULDUriIikkJyckAjsuy/MmAFNmsQdkVRHqjkQEUkR2dlwzjlw8cXhCgUlBhIXJQciIikgOxt+9SsYOxYOPFDNCBKvpCUHZra/mU0ys7lm9oWZXRmV32xmy81sRvQ4KWGdP5jZQjP70sx+mqzYRERSye7d8MtfwgsvwN/+BiNHxh2RVHfJ7HOQDVzr7tPNrCEwzczejub93d3vTlzYzLoDw4AeQGvgHTPr7O45SYxRRCR2554L48bBPffA1VfHHY1IEpMDd88EMqPpzWY2D2izh1WGAM+7+07gGzNbCPQBPklWjCIiqWDECOjbV+MYSOqokD4HZpYBHAZ8FhVdYWazzGy0mTWNytoASxNWW8aekwkRkUpr5054880wPXCgEgNJLUlPDsysAfAv4Cp33wQ8BBwAHEqoWRhVyu1dbGZTzWxqVlZWuccrIpJsO3bA6afDSSfBV1/FHY3IDyU1OTCzmoTE4Fl3fxnA3Ve5e4675wKPEZoOAJYD+yes3jYqy8fdH3X33u7eu0WLFskMX0Sk3G3fDkOGwBtvwMMPQ+fOcUck8kPJvFrBgCeAee5+T0J5q4TFTgfmRNPjgWFmVtvMOgCdgCnJik9EpKJt2wanngpvvw1PPBGGRxZJRcm8WuFoYDgw28xmRGV/BH5pZocCDiwGLgFw9y/M7EVgLuFKh8t1pYKIVCX//je89x489VQY7EgkVZm7xx1DmfXu3dunTp0adxgiIiU2Zw707BlvDGY2zd17xxuFpDKNkCgikkSbN4c+BtOnh9dxJwYiJaHkQEQkSTZtgkGD4PXXYdGiuKMRKTndlVFEJAk2bgyJwdSpYVjkM86IOyKRklNyICJSzjZuDAMbTZ8OL74YxjQQqUzUrCAiUs7q1IG2beFf/1JiIJWTag5ERMrJ2rXgDs2bh8RApLJSciAiUg7WrIEBA6BuXZg8Gczijkik7JQciIjspaws6N8fFiyA8eOVGEjlp+RARGQvrFoVEoNFi2DChDAtUtkpORAR2QsXXQTffBPGMjjuuLijESkfSg5ERPbCgw/CkiVw9NFxRyJSfnQpo4hIKS1fDiNHQk5OuGRRiYFUNUoORERKYelS+MlPQo3BggVxRyOSHEoORERKaMmSkBhkZcFbb0HXrnFHJJIc6nMgIlICixeHDofr18Pbb0OfPnFHJJI8Sg5EREpg6VLYvRvefRd69Yo7GpHkUnIgIrIHW7ZAgwZwzDGwcGG4b4JIVac+ByIiRfjqK+jWDUaPDq+VGEh1oZoDEZFCzJ8Pxx8P2dlwxBFxRyNSsZQciIgUMG9e6HzoDpMmQY8ecUckUrHUrCAikmDdOujXL9w86f33lRhI9aTkQEQkQbNmcNNNITHo1i3uaETioWYFERFgxgzYsQOOPBIuuyzuaETipeRARKq96dNhwABo0wZmzoQ01alKNac/ARGp1qZOhf79oWFDePVVJQYioORARKqxKVNCjUGTJvDBB9CxY9wRiaQGJQciUm099BDss09IDDIy4o5GJHWoz4GIVDvu4VLFRx+FtWthv/3ijkgktSSt5sDM9jezSWY218y+MLMro/JmZva2mS2InptG5WZm95nZQjObZWaHJys2Eam+Pv4Y+vaF1auhZk0lBiKFSWazQjZwrbt3B44ELjez7sBI4F137wS8G70GOBHoFD0uBh5KYmwiUg198AEMGgQbN0JOTtzRiKSupCUH7p7p7tOj6c3APKANMAR4OlrsaeC0aHoIMMaDT4EmZtYqWfGJSPUyaRKcdBK0bx+mW+m/i0iRKqRDopllAIcBnwEt3T0zmrUSaBlNtwGWJqy2LCoruK2LzWyqmU3NyspKWswiUnV8+CEMHgwdOoTEQE0JInuW9OTAzBoA/wKucvdNifPc3QEvzfbc/VF37+3uvVu0aFGOkYpIVdWpU0gOJk2CffeNOxqR1JfU5MDMahISg2fd/eWoeFVec0H0vDoqXw7sn7B626hMRKRMpk4Nt1xu1Qpeegn0e0KkZJJ5tYIBTwDz3P2ehFnjgRHR9Ajg1YTyc6KrFo4ENiY0P4iIlMqECXD00XDLLXFHIlL5JHOcg6OB4cBsM5sRlf0RuBN40cwuAJYAQ6N5E4GTgIXANuC8JMYmIlXY+PHw85/DIYfANdfEHY1I5ZO05MDdPwasiNn9C1negcuTFY+IVA+vvAJDh0KvXvCf/4ShkUWkdDR8sohUGRs3wgUXwBFHwJtvKjEQKSsNnywiVUbjxvD229C5c7jLooiUjWoORKTSGzsW7rsvTPfqpcRAZG8pORCRSu2f/4Thw0Nfg+zsuKMRqRqUHIhIpfXUUzBiBPTrFy5dTFdDqUi5UHIgIpXS6NFw/vkwYAC89hrUrx93RCJVh5IDEamUtm2Dn/4UXn0V6tWLOxqRqkXJgYhUKqtWhecrroDXX4e6deONR6QqUnIgIpXGAw/AgQfCrFnhdZr+g4kkhf60RKRSuPde+M1voH9/6No17mhEqjYlByKS8u65B666Cn72s3B3xVq14o5IpGpTciAiKW3CBLj2WjjzTHj+eahZM+6IRKo+JQciktIGDQp9DcaOVWIgUlGUHIhISnrwQVixIgxsdPnlGuBIpCIpORCRlOION98cEoIHH4w7GpHqSbm4iKQMd7jxRrjtNjj3XLjllrgjEqmeVHMgIinBHf70p5AYXHghPPEE1KgRd1Qi1ZOSAxFJCVu3hnskXHIJPPKIBjgSiZOaFUQkVu6QkwMNGsDHH0PDhkoMROKmP0ERiY07XH11GMMgOxsaN1ZiIJIK9GcoIrFwh9/+NgyLnJGh/gUiqUTJgYhUuNzccKniAw+E0Q/vuQfM4o5KRPIoORCRCvf738NDD8H118NddykxEEk16pAoIhVu6NDQv+DPf1ZiIJKKVHMgIhUiJwdefz1M9+kDN9ygxEAkVRWbHJhZXzP7XzObZWZZZvatmU00s8vNrHFFBCkilVtOThjx8OST4ZNP4o5GRIqzx+TAzN4ALgTeBAYBrYDuwJ+BOsCrZnZqsoMUkcorOxuGD4dnngmjH/btG3dEIlKc4vocDHf3NQXKtgDTo8coM2uelMhEpNLLzoZf/QpeeAHuuANGjow7IhEpiT3WHOQlBmZW38zSounOZnaqmdVMXKYgMxttZqvNbE5C2c1mttzMZkSPkxLm/cHMFprZl2b20/I4OBGJ16RJITG46y4lBiKVSUmvVvgQOMbMmgJvAZ8DvwDO3sM6TwEPAGMKlP/d3e9OLDCz7sAwoAfQGnjHzDq7e04J4xORFHTCCTBrFhx0UNyRiEhplPRqBXP3bcDPgAfd/UzCibxI7v4hsK6E2x8CPO/uO939G2Ah0KeE64pICtm5E4YNg/feC6+VGIhUPiVODsysL6GmILoYibIOdnpFdOXD6KgmAqANsDRhmWVRWWGBXGxmU81salZWVhlDEJFk2LEDzjgjNCV89VXc0YhIWZU0ObgS+APwirt/YWYdgUll2N9DwAHAoUAmMKq0G3D3R929t7v3btGiRRlCEJFk2LEDTj89jGXw0ENw6aVxRyQiZVWiPgdRE8GHCa8XAb8t7c7cfVXetJk9BkyIXi4H9k9YtG1UJiKVwI4dMGQIvPUWPPooXHRR3BGJyN4obpyDx8ys0BbD6AqG881sT50SC67TKuHl6UDelQzjgWFmVtvMOgCdgCkl3a6IxKtmTWjTBp54QomBSFVQXM3B/wI3RAnCHCCLMPhRJ6ARMBp4trAVzew5oB/Q3MyWATcB/czsUMCBxcAlAFFTxYvAXCAbuFxXKoikvq1bYcOG7xMDDYcsUjWYuxe/kFkDoDdhhMTtwDx3/zLJsRWrd+/ePnXq1LjDEKmWtmwJwyFnZsLs2VCrVtwRSUmZ2TR37x13HJK6StrnYAvwfnJDEZHKYvNmGDwYJk8OwyIrMRCpWnTLZhEplU2b4MQT4bPP4Lnn4Mwz445IRMqbkgMRKZVrr4UpU8JYBmecEXc0IpIMJR3nAAAzq5esQESkcrjzzjCWgRIDkaqrRMmBmR1lZnOB+dHrQ8zswaRGJiIpY906uO66MDTyPvvAwIFxRyQiyVTSmoO/Az8F1gK4+0zg2GQFJSKpY+1aGDAA7r8fpk+POxoRqQglblZw96UFijQOgUgVt2YN9O8Pc+fCv/8NffvGHZGIVISSdkhcamZHAW5mNQn3WpiXvLBEJG5ZWSExWLAAxo9XU4JIdVLSmoNLgcsJd0pcTrhx0uXJCkpE4peZGfoaTJigxECkuinpIEhrCLdrFpEqbssWaNAADj4YFi6EOnXijkhEKlpJr1boYGb3mNnLZjY+75Hs4ESkYq1YAb17w//8T3itxECkeippn4N/A08ArwG5yQtHROKyfDkcd1xoTjj66LijEZE4lTQ52OHu9yU1EhGJzdKlITFYvRrefBOOOiruiEQkTiVNDu41s5uAt4CdeYXurqueRSq5HTvg+OPD1QlvvQVHHhl3RCISt5ImBwcBw4Hj+b5ZwaPXIlKJ1akDN94IXbpAnz5xRyMiqaCkycGZQEd335XMYESk4nz9NXzzTRj9cPjwuKMRkVRS0uRgDtAEWJ3EWESkgixcGPoYuOtyRRH5oZImB02A+Wb2Ofn7HJyalKhEJGm++iokBrt2wTvvKDEQkR8qaXJwU1KjEJEKMX9+6HyYnQ2TJkHPnnFHJCKpqKQjJH6Q7EBEJPmeegpyckJi0KNH3NGISKra4wiJZvZx9LzZzDYlPDab2aaKCVFE9pZ7eL79dpg2TYmBiOxZccMn1wdw94bu3ijh0dDdG1VAfCKyl2bNCpcoLl4MaWnQtm3cEYlIqiuuWcErJAoRSYoZM8KlinXqwO7dcUcjIpVFccnBvmZ2TVEz3f2eco5HRMrJ9OkhMWjQIPQxOOCAuCMSkcqiuOSgBtAAsAqIRUTKycyZ0L8/NGoUEoOOHeOOSEQqk+KSg0x3v7VCIhGRctO+fag1uOsuyMiIOxoRqWyKSw5UYyBSicyaBZ06QZMm8NJLcUcjIpVVcVcr9K+QKERkr02eDD/+MVx5ZdyRiEhlt8fkwN3XlXXDZjbazFab2ZyEsmZm9raZLYiem0blZmb3mdlCM5tlZoeXdb8i1dFHH8FPfwqtWsFNGs9URPZScTUHe+MpYFCBspHAu+7eCXg3eg1wItApelwMPJTEuESqlA8+gBNPDOMXvP8+tGkTd0QiUtklLTlw9w+BgjUPQ4Cno+mngdMSysd48CnQxMxaJSs2kapi1y4YMSJ0QJw0KdQciIjsrZLeeKm8tHT3zGh6JdAymm4DLE1YbllUlkkBZnYxoXaBdu3aJS9SkUqgVi2YMAH23Tc8RETKQzKbFfbI3Z0yjMDo7o+6e293792iRYskRCaS+t58E26+OdwzoWdPJQYiUr4qOjlYlddcED2vjsqXA/snLNc2KhORAiZOhCFD4N//hm3b4o5GRKqiik4OxgMjoukRwKsJ5edEVy0cCWxMaH4QkciECXD66eGuiu+9B/Xrxx2RiFRFSetzYGbPAf2A5ma2DLgJuBN40cwuAJYAQ6PFJwInAQuBbcB5yYpLpLJ69VU480w45BB46y1o2jTuiESkqkpacuDuvyxi1g8GVor6H1yerFhEqoLt2+GII+D118MIiCIiyRJbh0QRKZnVUc+cYcPCYEdKDEQk2ZQciKSwF14IN0764IPwOk1/sSJSAfSvRiRFjR0LZ50VmhJ69Yo7GhGpTpQciKSgZ56B4cPh2GPDpYsNGsQdkYhUJ0oORFLMZ5/BOedAv36h86EuVxSRiqbkQCTF9OkDDzwAr70G9erFHY2IVEdKDkRSxJgxsGABmMFllykxEJH4KDkQSQEPPRTurvi3v8UdiYiIkgOR2D3wQKgpOOWUMC0iEjclByIx+sc/4De/gdNOg3HjoHbtuCMSEVFyIBKb7Gx46SU44wx48UWoVSvuiEREgqTdW0FEirZ7N9SsCf/5D9SpE6ZFRFKFag5EKtjtt8MJJ8C2bdCwoRIDEUk9Sg5EKtBf/gJ/+hO0aaNmBBFJXUoORCqAO9x8M9x4YxgWecwYSFejnoikKCUHIhXg7rvhllvg3HPhySehRo24IxIRKZp+u4hUgJNPhqwsuPNO3XZZRFKf/k2JJIk7TJgQnrt1C6MfKjEQkcpA/6pEksAdrrsujHr46qtxRyMiUjpqVhApZ+5w9dVw771h9MMhQ+KOSESkdFRzIFKO3OG3vw2JwVVXhWezuKMSESkdJQci5WjGjHCHxeuug3vuUWIgIpWTmhVEysGOHeHyxMMOg+nT4aCDlBiISOWl5EBkL8ydC489Bk8/DY88AmeeCQcfHHdUIqlv2rRp+6anpz8O9ES12BUtF5iTnZ19Ya9evVYXtoCSA5FSysmBsWPh0Ufh44/DvRF+9jPo2DHuyEQqj/T09Mf322+/bi1atFiflpbmccdTneTm5lpWVlb3lStXPg6cWtgySg5ESmj1ath33zBWwW23hc6Hd90FI0ZAixZxRydS6fRUYhCPtLQ0b9GixcaVK1f2LGoZJQcie7BtG7z4YqglmDcPli+HevVg0iRo1Ur9CkT2QpoSg/hE732RzTlq5xEpxDffwBVXQOvWcN55sG4d3HBDqC2AUK7EQESqqliSAzNbbGazzWyGmU2NypqZ2dtmtiB6bhpHbFJ9bd0Kq1aF6awsePzxcE+EDz4ItQbXXAP168cbo4iUjy+//LJWp06desQdR1HOOOOMjDZt2hzUtWvX7l27du0+efLkugC5ubmce+65+7dr165n586du3/88cf1krH/OJsVjnP3NQmvRwLvuvudZiJXTPYAABjOSURBVDYyen19PKFJdTJjRmg2eOYZ+MUvwtUHRxwBmZnQVCmqiMTktttuW3beeeetTyx76aWXGi9atKjO4sWL50yaNKn+ZZdd1m7WrFnzy3vfqdSsMAR4Opp+GjgtxlikGhg7Fvr0CWMTPPkknHZaaEKA0GSgxECkepg7d26tbt26df/ggw/qZWdnc8kll7Tt2bNnt86dO3e/6667mkP4xX7JJZe07dSpU4/OnTt3f+yxx5oCTJgwoWHv3r279OvX78CMjIyeZ511VrucnByys7M544wzMvKWv+WWW/Ytj1hfffXVJmefffbatLQ0+vfvv3XTpk3pS5YsqVke204UV82BA2+ZmQOPuPujQEt3z4zmrwRaFraimV0MXAzQrl27iohVqpCZM8M4BGbwySewfTvcdx/86ldKBkTicv757D9nDuVaPd6zJ9tGj2ZpccvNnDmz9rBhww4YPXr0N3379t1+9913N2/cuHHOnDlz5m3fvt2OOOKIrqeccsqmTz/9tN7s2bPrzps374vMzMz0Pn36dBs4cOAWgNmzZ9f/73//O6dz5867jj322E5jxoxpeuCBB+7MzMysuWDBgi8A1qxZU6O0x3DLLbe0ueOOO1odc8wxmx944IFldevW9czMzJoZGRm78pZp1arVriVLltRs37797tJuf0/iqjn4sbsfDpwIXG5mxybOdHcnJBA/4O6Puntvd+/dQtePSQls3hyaDXr3hkMPhQ8/DOV33QWzZoWbIykxEKl+1q1bl37aaacd+Mwzzyzq27fvdoB33nmn0YsvvrhP165dux922GHd1q9fnz537tw6H330UcOhQ4euS09PZ//998/+0Y9+tCWvvf+ggw7a2r17913p6ekMHTp03UcffdSga9euO5cuXVp7xIgR+48bN65R06ZNc0oT2z333LN80aJFc2bOnDlv/fr1NW644Yb9kvEeFCWWmgN3Xx49rzazV4A+wCoza+XumWbWCih01CaRklq3DkaODM0HW7eGIY0feAAOOSTMr1Mn3vhEJCjJL/xkaNiwYU7r1q13TZo0qUGvXr12ALi7jRo16tszzjhjU+Kyr7/+euOitmMFLl0yM1q0aJEzZ86cua+88kqjhx9+uMULL7zQ7KWXXlqct0x2djY9e/bsDjBo0KAN//jHP1YkbiOvJqBu3bp+/vnnrx01alRLgFatWu1evHhxrbzlMjMza5V3rQHEUHNgZvXNrGHeNDAQmAOMB0ZEi40AXq3o2KTy27gRpk0L0w0awDvvhE6Gn34amhQuvxyaNIk3RhFJDTVr1vQ33njj6+eee26fhx9+uBnACSecsPGhhx5qsXPnTgOYNWtW7U2bNqUde+yxm8eNG9csOzubFStWpE+ZMqXBMcccsxVCs8L8+fNr5eTkMG7cuGbHHHPM5szMzPScnBzOPffcDXfcccfy2bNn52s2SU9PZ/78+XPnz58/t2BiAJDXjyA3N5eXX365Sbdu3bYDnHrqqRueffbZfXJzc3n33XfrN2zYMCcZyUEcNQctgVeiTCsdGOvu/zGzz4EXzewCYAkwNIbYpBJyhylTQtPB88+H0Qq//hpq1YKvvoJ0DfUlIkVo1KhR7ptvvrmwX79+nRs2bJhz9dVXr1m8eHHtgw46qJu7W7NmzXZPnDjx6+HDh2+YPHlyg27duvUwM7/llluWtWvXLnvWrFn07Nlz66WXXtpu8eLFdY466qhNw4cP3zBlypS6F1xwQUZubq4B3HrrrctKE9cvfvGLDuvWrUt3d+vevfu2MWPGLAEYOnToxtdff71x+/bte9atWzf38ccfX5yEtwVzr7wDVPXu3dunTp0adxgSo4kT4Q9/CH0H6teHX/4SLr449C/QIEUihTOzae7eO84YZs6cufiQQw5ZU/ySqW3ChAkNR40a1XLSpEkL446ltGbOnNn8kEMOyShsnn5TSaXiHpoI2rWDNm0gNzfc+OiRR0Ji0LBh3BGKiFR+qTTOgUiR1q8PlxwedBAcdRQ8/HAoHzwYpk4NtQVKDESkop188smbK2OtQXFUcyApzR0uugiefRZ27AiDFj3+eOhkCGo6EBFJBtUcSMpZty50LIRw8jcLIxf+97/w2WdwwQXhSgQREUkO1RxISnCHjz4KVxyMGwc7d4Zago4dw70ORESk4qjmQGI3YwZ06wY/+QlMmBCaEWbODImBiIhUPCUHUuHcYdIkeO+98DojA1q3hqeeghUr4P77w/0PRERSyciRI8ttCONrrrmm9Y033ljoPYRKqkaNGr3ybul8/PHHH5hXPn/+/FoHH3xw13bt2vUcPHhwxx07dpS6d5aSA6kwq1eH+xl06QLHHw9//Wsob9IkJAojRkC9pNyZXERk7913332tSrN8bm4uOTmluqVCqdSuXTs3b5TF995777srJq655pq2V1xxxapvv/12TuPGjbPvvffe5qXdtpIDqRA33QRt28Lvfw8tW8KYMaEJQUQkDl9++WWtDh069Dj11FM7dOzYscegQYM6bt68OW38+PENBwwYcEDecq+88kqjE0444YDLLruszc6dO9O6du3a/dRTT+0AcPPNN7fs1KlTj06dOvW49dZb983bbkZGRs/TTz89o3Pnzj2+/vrrWuPGjWvUvXv3bl26dOnet2/fznnbnjdvXt0+ffp0adu27UG33XZbudzSOTc3l08++aTheeedtx7g/PPPX/vaa6+VetB4dUiUpFi1KjQTXHQRNGsW+hRcfnl43b173NGJSKrp04cuBct+9jPWjRxJ1ubNpPXvT6eC83/1K9b89reszcwkfcgQDkicN2UKXxa3z8WLF9d55JFHFg8cOHDrmWeemXHXXXe1uPnmm1ddeeWV7VasWJHeunXr7NGjR+9z3nnnrTnrrLM2PvXUU/vOnz9/LsBHH31Ub+zYsftMmzZtnrvTq1evbv3799/cvHnznG+//bb2E0888U3//v0Xr1ixIv2KK67IeP/99+d37dp116pVq767dfPChQvrTJ48+csNGzbU6NatW8/f/e53WbVr1y7xsMW7du1K69mzZ7caNWr4ddddt3L48OEbVq1ald6wYcOcmjVrApCRkbFr1apVtYrZ1A+o5kDKTW4uvPUW/PznoZZg5Eh4++0wb9gw+PvflRiISOrYb7/9dg0cOHArwPDhw9dOnjy5QVpaGkOHDl372GOPNVuzZk2N6dOnNzjzzDM3Flz3/fffb3DSSSdtaNSoUW7jxo1zBw8evH7SpEkNAVq1arWrf//+W6Pl6vfp02dz165ddwG0bNnyu3aGgQMHbqhbt663atUqu1mzZruXLVtWqh/sCxYsmDVnzpx5zz333KKRI0fu/8UXX9Tem/cjkWoOpFxs3gyHHgqLFsE++8CVV8KFF0LXrnFHJiKVwZ5+6TdsSO6e5rdqRXZJagoKKuxWywC//vWv1w4ePPjAOnXq+CmnnLI+71d4SdWrVy+3JMsl1hLUqFGD7OzsfAGNGTOmye23394a4NFHH1187LHHbkuc36FDh90A3bt333XkkUdunjJlSr0RI0as37x5c43du3dTs2ZNFi9eXKtly5a7SnUAqOZAyig3F958E+6+O7xu2BBOOw2eew6WLw/lSgxEJJVlZmbWeuedd+oDPPvss82OOuqoLQAZGRm7W7ZsuXvUqFGtLr744u9uDpWenu55t3I+7rjjtkycOLHJ5s2b0zZt2pQ2ceLEpscdd9zmgvvo16/f1ilTpjScP39+LYDEZoXinHPOORvyOhwWTAyysrJqbN++3aLjSJ86dWqDgw8+eHtaWhpHHnnk5ieffLIpwOjRo/c5+eSTN5T2vVFyIKWyYgXcdlsYg2DQoNBUsGNHmDdqVGg+qF1uFVsiIsmTkZGx4/7779+3Y8eOPTZs2JB+3XXXZeXNGzZs2NpWrVrtOvzww3fklZ199tlZ3bp1637qqad2+PGPf7ztrLPOWnv44Yd369WrV7fhw4dnHX300dsL7qN169bZ99133+LTTz/9wC5dunQ//fTTy2UElxkzZtQ55JBDunXp0qX7T37yk85XXXXVyl69eu0AGDVq1LL7779/v3bt2vVcv359+pVXXlnqu1/qls1SYmPHwjnnQE4O9O8fbnY0ZIiSAZHKRrdsDlcVnHzyyZ0WLFjwRWHzzznnnHaHHXbYtquvvrrS31a6KLpls5TJsmUwenS4C+KAAfDjH8Pvfhf6EhxwQPHri4hURj169OhWt27d3EceeWRp3LHERcmB5JOTA2+8Ee5x8PrroW/Bn/8ckoN27eCOO+KOUERk73Xp0mVXUbUGX3zxxbyKjifVKDmQfI4/Hj78EPbbL1yKeMEFuseBiCRFbm5urqWlpVXetu1KLDc314Air6pQh8RqLDsbXn01dCLcuTOUXXEFvPwyfPttGN5YiYGIJMmcrKysxtFJSipQbm6uZWVlNQbmFLWMag6qocWL4YknQn+CFSugVSv46is46CA488y4oxOR6iA7O/vClStXPr5y5cqe6IdqRcsF5mRnZ19Y1AJKDqqZL74ISQDAiSfCgw/C4MGQrm+CiFSgXr16rQZOjTsOKZxOCVXcN9/A44+Hk/8tt4Thi++5B372s9DBUEREpCAlB1XQxo3hngaPPRbudZCWBr/8ZZhnBlddFW98IiKS2tTOU4llZ8PcufD88/DHP8LWraH89ttD34H58+HWW2HJEnjmmXhjFRGRykM1B5XEqlXQoAHUrx9qBa6/PvQf2BXdTiM9PVx1cPDBYRTDgQOhXz+oUeJRvEVERAIlBylo3Tp47TWYNev7x+rV8Mor4eZGDRrAvvuGgYkOPjg8unaFWtEdu3v0CA8RSW25ubB79/eP7Oz8r/c0LyMDOneO+wikqlJyEBP3MDxxYgJw2mnwi1/AmjVw7rlQpw707Aknn/x9EgDQty/85z+xhi9S4dzLdhLd23nJ3FduiW7sW7jrr4c77yy/91ckkZKDCrBlC8yZE6r4jzgi3MWwdWtYv/77ZTIy4JhjwvQBB4T+AgceqGYBKRv3709AcZz0krG9nJyKe//S0qBmze8f6en5XxdVXq9e8euUZnt7mrf//hX3fkj1k3LJgZkNAu4FagCPu3tK58busHlzGExo167vf93ffjt8/jnMng1ffx3KTjkFxo8PNQKXXhr+uA8+ONQONG78/TZr1IAuXSr+WCob9/DIyQm/wHJzv58uaVl5r5MKJ9G81xXFrGwnvTp1yn4SLa8TbFHz0tRVW6q5lEoOzKwG8L/ACcAy4HMzG+/uc8tzP2vWhBEBIZxcEp/zpnfuhG3bYMMGyMqCtWvDP91Bg8L8e++FadNC/4C8oYcPOCCUu8MLL4R12reHPn3Cc7t28NJLYf7BB4fnJUvCWAR5J7rc3D1PFzc/cbosJ8NUOMGWdJ3KerfxspzA6tcv/5NoeZ1gVbslUvWkVHIA9AEWuvsiADN7HhgClGtyMGkSDB1atnXvvrvoeV9/HfoHJFq+HCZPLtu+yluNGuEXUVra99OFlRU3v7h10tPLf5vltU6y4yjuBFujRvilLSKSylItOWgDJN4/exnwo8QFzOxi4GKAdmUc4u+YY+Caa2DGjB9Wc55/fljm88/Dib1JE9hnH2jePDzq1w//3Ev6SEvL/7w306VdL/HkpROSiIiUVKolB8Vy90eBRwF69+5dporl/faDUaP2vMyAAWXZsoiISOWXat1ulgOJfXDbRmUiIiJSQVItOfgc6GRmHcysFjAMGB9zTCIiItVKSjUruHu2mV0BvEm4lHG0u38Rc1giIiLVSkolBwDuPhGYGHccIiIi1VWqNSuIiIhIzJQciIiISD5KDkRERCQfJQciIiKSj3llHaAeMLMsYEkxizUH1lRAOKlGx139VNdj13GXXnt3b1GewUjVUqmTg5Iws6nu3jvuOCqajrv6qa7HruMWKX9qVhAREZF8lByIiIhIPtUhOXg07gBiouOufqrrseu4RcpZle9zICIiIqVTHWoOREREpBSUHIiIiEg+VTo5MLNBZvalmS00s5Fxx1NaZra/mU0ys7lm9oWZXRmVNzOzt81sQfTcNCo3M7svOt5ZZnZ4wrZGRMsvMLMRCeW9zGx2tM59ZmYVf6SFM7MaZvZfM5sQve5gZp9Fsb4Q3dYbM6sdvV4Yzc9I2MYfovIvzeynCeUp+90wsyZmNs7M5pvZPDPrWx0+czO7OvqezzGz58ysTlX8zM1stJmtNrM5CWVJ/3yL2odIody9Sj4It3z+GugI1AJmAt3jjquUx9AKODyabgh8BXQH/gaMjMpHAv8TTZ8EvAEYcCTwWVTeDFgUPTeNpptG86ZEy1q07olxH3fC8V8DjAUmRK9fBIZF0w8Dv46mLwMejqaHAS9E092jz7020CH6PtRI9e8G8DRwYTRdC2hS1T9zoA3wDVA34bM+typ+5sCxwOHAnISypH++Re1DDz0Ke1TlmoM+wEJ3X+Tuu4DngSExx1Qq7p7p7tOj6c3APMI/0SGEEwjR82nR9BBgjAefAk3MrBXwU+Btd1/n7uuBt4FB0bxG7v6puzswJmFbsTKztsBg4PHotQHHA+OiRQoed977MQ7oHy0/BHje3Xe6+zfAQsL3ImW/G2bWmHDyeALA3Xe5+waqwWdOuIV8XTNLB+oBmVTBz9zdPwTWFSiuiM+3qH2I/EBVTg7aAEsTXi+LyiqlqNr0MOAzoKW7Z0azVgIto+mijnlP5csKKU8F/wB+D+RGr/cBNrh7dvQ6Mdbvji+avzFavrTvRyroAGQBT0ZNKo+bWX2q+Gfu7suBu4FvCUnBRmAa1eMzh4r5fIvah8gPVOXkoMowswbAv4Cr3H1T4rzo10GVuh7VzE4GVrv7tLhjiUE6ocr5IXc/DNhKqAL+ThX9zJsSftl2AFoD9YFBsQYVk4r4fKvid0jKV1VODpYD+ye8bhuVVSpmVpOQGDzr7i9Hxaui6kOi59VReVHHvKfytoWUx+1o4FQzW0yo/j0euJdQpZoeLZMY63fHF81vDKyl9O9HKlgGLHP3z6LX4wjJQlX/zAcA37h7lrvvBl4mfA+qw2cOFfP5FrUPkR+oysnB50CnqLdzLUKnpfExx1QqURvqE8A8d78nYdZ4IK938gjg1YTyc6IezkcCG6NqxDeBgWbWNPqFNhB4M5q3ycyOjPZ1TsK2YuPuf3D3tu6eQfjc3nP3s4FJwM+jxQoed9778fNoeY/Kh0U92zsAnQidtVL2u+HuK4GlZtYlKuoPzKWKf+aE5oQjzaxeFFfecVf5zzxSEZ9vUfsQ+aG4e0Qm80Ho6fsVoZfyn+KOpwzx/5hQ9TcLmBE9TiK0rb4LLADeAZpFyxvwv9HxzgZ6J2zrfELnrIXAeQnlvYE50ToPEI2amSoPoB/fX63QkfCPfiHwElA7Kq8TvV4Yze+YsP6fomP7koRe+an83QAOBaZGn/u/Cb3Rq/xnDtwCzI9i+yfhioMq95kDzxH6Vewm1BRdUBGfb1H70EOPwh4aPllERETyqcrNCiIiIlIGSg5EREQkHyUHIiIiko+SAxEREclHyYGIiIjko+RAqiUzyzGzGWY208ymm9lRxSzfxMwuK8F23zez3mWMaaKZNSnLuiIi5UnJgVRX2939UHc/BPgDcEcxyzch3Akwadz9JA83WRIRiZWSAxFoBKyHcB8LM3s3qk2YbWZ5d+67Ezggqm24K1r2+miZmWZ2Z8L2zjSzKWb2lZkdU3BnZtbKzD6MtjUnbxkzW2xmzc3s0mjeDDP7xswmRfMHmtknUWwvRffcEBEpdxoESaolM8shjDhXB2gFHO/u0/JuF+zum8ysOfApYQje9oSRGntG658I3AAMcPdtZtbM3deZ2fvANHe/1sxOAq5x9wEF9n0tUMfd/2pmNaL9bY7uJdHb3ddEy9UE3gP+BnxCuN/Aie6+1cyuJ4wWeGsy3ycRqZ7Si19EpEra7u6HAphZX2CMmfUkDFd7u5kdS7hddBsKv7XtAOBJd98G4O7rEubl3SBrGpBRyLqfA6Ojk/+/3X1GETHeS7hnwGvRnSq7A/8XhsynFiFhEBEpd0oOpNpz90+iWoIWhPH3WwC93H139Gu+Tik3uTN6zqGQvzF3/zBKPgYDT5nZPe4+JnEZMzuXUFtxRV4R8La7/7KUsYiIlJr6HEi1Z2ZdgRqEW/42BlZHicFxhBM0wGagYcJqbwPnmVm9aBvNSrG/9sAqd38MeJxwS+bE+b2A64BfuXtuVPwpcLSZHRgtU9/MOpfuSEVESkY1B1Jd1TWzvOp8A0a4e46ZPQu8ZmazCXdGnA/g7mvN7P/MbA7whrv/zswOBaaa2S5gIvDHEu67H/A7M9sNbCHcVjfRFUAzYFLUhDDV3S+MahOeM7Pa0XJ/JtxlUESkXKlDooiIiOSjZgURERHJR8mBiIiI5KPkQERERPJRciAiIiL5KDkQERGRfJQciIiISD5KDkRERCSf/wfym1pzeYC0sAAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "max_batch_size = 100000\n", + "\n", + "plot_absolute_time(experiments_eq, results, [50], max_batch_size=max_batch_size)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiIAAAEWCAYAAABbt/wMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd5gUVdbH8e8hgyRBFhGEwUAYsyJrXAOKKAgqopgxYQ5g3HXXdV3X8LrmHEDEhCsmRFwjiFlBRAEBUREkSc4Cw5z3j3tH29kJPTg9NeH3eZ5+ZupWOhW6+/StW7fM3RERERFJQrWkAxAREZGqS4mIiIiIJEaJiIiIiCRGiYiIiIgkRomIiIiIJEaJiIiIiCSmwiYiZuZmtl0R4x80s7+lDJ9nZgvMbJWZNd2E9WXFddbY1JhLS9yGbZKOo6yY2RAzu6EE0xd5bpQ3ZvaamZ2WdBzpMLPW8fyrvgnz1jWzV8xsuZk9l4n4ZNOY2Ulm9kbScUjVVGwiYmYzzWy9mW2Rr3xC/MDPisND4nCvfNPdEcv7xeF+ZrYxfpitMrPvzewxM2tXalsFuPu57v7PuM6awO1AV3ev7+6LM/llFbfx/VJa1hgzOyu1LG7Dd6Wx/NJQHpI0M9vbzD5Mav3pMrPrzOzJ1DJ3P9zdH08qpqLE9/8hecPuPiuefxs3YXHHAs2Bpu7epxRiq2Vmw2OMbmYH5htvZnaLmS2Or1vMzFLG72pm481sTfy76++NKQkl/bwp6P3q7k+5e9fMRChStHRrRL4HTsgbMLOdgHoFTDcdODVluhrAccC3+ab7yN3rA42AQ4C1wHgz2zH90AtXwK+15kAdYHJpLF9KVyklMN2BUaWwnCojgcSxDTDd3XNKOmMRsb4PnAzML2Bcf+AoYBdgZ+BI4Jy4vFrAy8CTwObA48DLsTxjYnJUYWuiRTLC3Yt8ATOBvwKfpZT9G7gGcCArlg2J5QuAzWNZD+A1wodFv1jWD3i/gPWMBIYXEccVwDxgLnBGXPd2Ket+gPBFtJqQ3AwBbgDaxTIHVgHvAGPj8OpYdnwB66set2cR8B1wQZynRhzfCBgUY5oT11Ud6Aj8DGyMy14Wp68dlzcr7qMHgbop6+sFfAGsICRu3YB/xeX8HJd1b5w2ddsbAUOBhcAP8VhVS93Xcb1LCQnl4cUc6z8DU+L0jwF14rhJwJEp09aM+2a3uE15+3cVsDchyf1rjOmnGGOjOG9WnP7MOO/YWL4f8CGwDJjNr+fMEOA+4FVgJfAJsG2+2D8Hdi9g/+wXl3VgHO4AvAksAaYBx6Uso7h9+QFwL7AcmAp0SZm3H+E8WRn380kF7N9uwHpgQ9xPE2P5GOCsfOu5I+6H74B9YvnsuC9PS1lmkedVvvWnLnsx4ZzdlvCeWByP51NA4zj9E0Au4YfCKuDKlGOX9z7YChgR9+cM4OxC1v2PfNt+JptwjhRx7v6Yd4xTyj4E+qcMnwl8HP/vSnjfWsr4WUC3QpY/BrgJ+JTwHn0ZaJIyfi9+PXcnpsYS5/1X3Pdrge1i2Q1xnlXAK0DTuP9XAJ/x62frb/Z56jlD4Z833YEJcVmzgevybWf+92s/Uj6XCefcZ4Rz/TNgn3zr/mfcnpXAG8AWcVwdQnK3OO6Lz4DmxX3P6FW1X8VPEL6cDiF8aHckfNn+SPh1kz8RuQF4GDgvlv2HUJOSTiJyBrCgkBi6ET5kdwQ2A57mfxOR5cC+hA+3OnnxxPEFvZF/mb+QdZ5L+LLZGmgCjOa3H8AvAg/FeP5A+IA6p7BtJHz4j4jLakD44Lkpjusc4z80xt8S6BDHjSF+SRUUO+HD++W4zCxCrdSZKXFsAM6Ox+08QiJnhWzzTELCkbfNH6TswyuBZ1Om7QV8VcT+PYPwxbQNUB94AXgi3/RD4/6rSzifVsbzpSbhQ3nXlOO7OO6nGoQP62Ep62pBypdK3v4hnDezgc6xfLM4fHpczm6EL9/sNPdlDjAgxnd8PGZN4nJXAO1T4tmhkH18HfBkvrJfjnHKek6Px+wGwhfHfYSko2vcT/WLO68KWHfesi+K21837qdD47KbEZL0O/O//1OGf3Os4/T3E95zuxKSuIPT2XZKeI4U8zlVUCKyHPhjynAnYGX8fwDwWr7pRwKXFbL8MYRzLO8z6Pm8bSG8XxcDRxDev4fG4WYp884Cdoj7vWYsm0FIBBsRkv/phM/aGnG7Hyvi/ZX/nMn/eXMgsFOMZ2fC5+dRRSzvl2UQzqWlwCkxlhPicNOUdX9L+JFXNw7fHMedQzgH6xHO3z2AhsV9z+hVtV8lqSJ8gnDZ5VDga8KbsiBDgVPNrDFwAPBSmsufS3gDFOQ4wptykruvJnyg5feyu3/g7rnu/nOa6yzKcYQP5NnuvoTwawgAM2tO+NC51N1Xu/tPhC+EvgUtKF6X7g8McPcl7r4SuDFl+jOBwe7+Zox/jrtPLS7AeAmqL/Bnd1/p7jOB2wgfIHl+cPdHPFzTf5zwJdm8iMXem7LN/+LXS3JPAkeYWcM4fArhnCjMScDt7v6du68i1LT0zVfFfl3cf2uBE4G33P0Zd9/g7ovd/YuUaV909089VOs/RfjSy3ME8F9395SyPoRE8XB3/zSW9QBmuvtj7p7j7hMIXyh90tyXPxHOiQ3u/iwhOe8ex+UCO5pZXXef5+6/5zLg9zHGjcCzhMTwendf5+5vEGoWtkvjvCrIXHe/J27/WnefEc+7de6+kNCW6oB0gjSzrQnJ/1Xu/nM8Xo+Scnm2GCU9R0qqPiEZybMcqB/3W/5xeeMbFLG8J1I+g/4GHBfPm5OBUe4+Kr5/3wTGEc7LPEPcfXLc7xti2WPu/q27LyfUHH/r7m/Fc/w5QqK8Sdx9jLt/FeP5EniGNI8r4Zz+xt2fiPE+Q/hRdmTKNI+5+/R4XP7Dr+/HDYQfEdu5+0Z3H+/uKzZ1O6RqKMk14icIv37aEpKNArn7+2bWjHDpZqS7r01pH1aUloTq3YJsBYxPGf6hgGlmp7OSEtgq3zJT19mG8KtmXsq2VSsihmaEXwjjU9vKEX4xQPii2ZT2DVvEOFJj+4GwL/P8cu3c3dfE9dcvYpn5t3mrOO9cM/sA6G1mLwKHA5cUsZytCoirBr9NglLXtTX/25YoVWobgDX8dhuOINSSpboUGOruk1LK2gB/NLNlKWU1COd2OvtyTr5k5wdgK3dfbWbHA5cDg+J+uiydZLIQC1L+Xwvg7vnL6lP8eVWQ35yjMam+C9if8CVcjfDrNx1bAXkJUJ4fCDUP6c5fknOkpFYBDVOGGwKr3N3NLP+4vPErKVz+90ZNwnnThpDMpn5R1yTUohY0b578x7SgY7xJzOyPwM2EGpxahBqvdO9Uyn9coIjPFX77fnyC8F4eFn+MPglck5J8ifyPtGtE3P0HwrXvIwhVqEV5EriMIhKWAhwNvFfIuHmEkztP64JCLMG60lHUOmcD6wjXRRvHV0N336GQWBYRPlh2SJm+kYcGu3nL27aQOIrarkWEXyBt8sVZWG1VOvJv89yU4ccJv/76EBoc562noBjnFhBXDr/9sE2dr6h9UKh4R9QBhHYfqfoAR5lZarI0G3g35Rg09nAHyHmkty9b2m+z6l/2j7u/7u6HEmqcpgKPFBJyaZ6nxZ1X6az/xli2k7s3JBxfK2L6VHOBJmaWWotQkvOvpOdISU0mNFTNswu/NlifDOyc73juTNEN2vO/NzYQjsFsQm1J6nm1mbvfnDL979mO1fFv6g0CWxaz7KcJl+y2dvdGhLZDVsT0qfIfF0jzuMbawn+4ezahnUkP0q8hkyqqpK23zyRc/11dzHR3Ey7hjC1qIjOrbmZtzewewjXNfxQy6X+AfmaWbWb1gL+XLOwCLSBcmy7Mf4CLzayVmW0OXJ03wt3nERpo3WZmDc2smplta2Z5VZ8LgFZ5LfDdPZfwxXSHmf0BwMxamtlhcfpBwOlm1iUuq6WZdSguzlh1/x/gX2bWwMzaAAMJieCmuiBucxNCrdazKeNeAnYn1ISkJpkLCZcmUuN8BhgQj299whfes174HRNPAYeY2XFmVsPMmqZ5O+V+wJcFVP/OBboAl5jZebFsJNDOzE4xs5rxtaeZdUxzX/6BcE7UNLM+hDZTo8ysuZn1MrPNCAnqqrg/CrIAyCqNOyfSOK/S0YAQ73Iza0loFJ4/3sLOv9mExpY3mVkdM9uZ8BmR7vlX0nPkf5hZbTOrEwdrxTjyvnCHAgPjPtmK8ONoSBw3htDA8+K4jAtj+TtFrO7klM+g6wmN6zcStvdIMzssfqbVMbMDzaxVuttRlHjJbE5cf3UzO4PfJu2/+byJGhBqq342s86ES595Cnq/phpFeJ+cGN+LxwPZhPdPkczsIDPbKV6yWkFI1gp7L4gAJUxE4vXMcWlMt8Td385XjZ1q71g1uoLwgdAQ2NPdvypkea8BdxI+JGZQ9IdFuq4DHjezZWZ2XAHjHwFeJ7SA/5z/rQU6lVDlmXeHyXDCr2FifJOB+Wa2KJZdFWP/2MxWAG8B7eP2fUponHgH4Tr1u/z6i+Qu4FgzW2pmdxcQ50WEX0zfERoFPw0MTm8XFOhpQpL1HeFSyS8dicXrwc8TLs+9kFK+hnhXQNyfe8UY8i7nfU9o2X9RYSt191mE2rbLCJfovuC3v2YLU+htu3GZXYCrzeyseAmhK6ENxVxC9fIthGprKH5ffgJsT/gV/C/gWHdfTHgfDYzLXEKooTmPguVVjy82s8/T2L7iFHpepekfhORyOeGupPzn+U3AX+NxvbyA+U8gNH6cS2jA/Xd3fyvNdZfoHCnENEKtUEvC+3Utv753HiI0nPyK0Aj71ViGu68n3Np7KuHujjMIjTnXF7GuJwiJzHxC49yL47JmExpv/4XwJT+bkNCV5m26Z8dlLiY0ek3tM6egz5vzgevNbCVwLSHJJsZb0PuVlPGLCTUZl8X1XQn0cPdFFG9LwmfhCkJbwncpui2ZyC93GYhgZjMJLfEL/SIxs2uBdu5+cpkFVgQzm0JICKZkeD39CPtmv0yuR8onMxtDuEvm0aRjEalsEu+uXCqOeLnmTH57J0liYlX00EwnISIikjnq4U/SYmZnE6qcX3P3Itv+lBV3X5+vQaCIiFQwujQjIiIiiVGNiIiIiCSmQrQR2WKLLTwrKyvpMEREKpTx48cvcvdmScchUpQKkYhkZWUxblyxdw2LiEgKMyuoF2qRckWXZkRERCQxSkREREQkMUpEREREJDFKRERERCQxSkREREQkMUpEREREJDFKRERERCQxSkRERMqhn3+GQYPgmWeSjkQks5SIiIiUIwsWwN//Dq1bw1lnwbBhSUckkllKREREyoGJE6Ffv5CAXH89/PGP8Pbb8NJLSUcmklkVoot3EZHKKDcXXn0V7rgDRo+GevVCLcgll0C7dklHJ1I2lIiIiJSxVatgyBC46y6YMQNatYJbbglJSJMmSUcnUraUiIiIlJHZs+Gee+CRR2DZMujcOTRG7d0batZMOjqRZCgRERHJsI8/hjvvhOHDwT0kHgMGwN57Jx2ZSPKUiIiIZEBODrzwQmj/8fHH0LAhXHopXHQRtGmTdHQi5YcSERGRUrRsWbj0cs894VLMttvC3XeHO2IaNEg6OpHyR4mIiEgpmDEjND597DFYvRoOOCAkIz16QPXqSUcnUn4pERER2UTuMGZMuPwyciTUqAEnnBAuwey2W9LRiVQMSkREREpo3brQ4+mdd8IXX8AWW8A118D550OLFklHJ1KxKBEREUnTwoXwwANw//2hK/bs7NAe5KSToG7dpKMTqZiUiIiIFGPSpFD78eSToTakW7dw++2hh4JZ0tGJVGxKRERECpCbC//9b0hA3nwz1Hj06xe6X+/YMenoRCoPJSIiIinWrIGhQ8MdMFOnhjYf//oXnHMONG2adHQilU/GExEzqw6MA+a4ew8zawsMA5oC44FT3H19puMQESnKnDlw333w0EOwZAnssUe4FNOnD9SqlXR0IpVXtTJYxyXA1ynDtwB3uPt2wFLgzDKIQUSkQOPGhcamWVlw882h/4+xY+Gzz0K5khCRzMpoImJmrYDuwKNx2ICDgeFxkseBozIZg4hIfhs3wvPPw/77w557wiuvwIUXhk7JXnghlKsRqkjZyPSlmTuBK4G8jo2bAsvcPScO/wi0zHAMIiIArFgBgwaFLtdnzgy1ILffDmeeGZ4FIyJlL2OJiJn1AH5y9/FmduAmzN8f6A/QunXrUo5ORKqS774LycfgwbByJey3H9x2G/Tqpe7XRZKWyRqRfYGeZnYEUAdoCNwFNDazGrFWpBUwp6CZ3f1h4GGATp06eQbjFJFKyB3efz90v/7yy1CtGhx3XOj/o1OnpKMTkTwZayPi7n9291bungX0Bd5x95OA0cCxcbLTgJczFYOIVD3r14e7XfbcE/70J3j3XbjqqnAp5qmnlISIlDdJ9CNyFTDMzG4AJgCDEohBRCqZxYvDrbf33Qdz50L79qE79lNPhXr1ko5ORApTJomIu48BxsT/vwM6l8V6RaTy+/rr0PnY0KGwdm3odv3RR+Gww8LlGBEp39SzqohUOO6h2/U77gjdsNeuDSefDJdeCjvumHR0IlISSkREpMJYuza087jzTpg8GZo3h+uvh3PPhWbNko5ORDaFEhERKffmzYP774cHH4RFi2CXXWDIEOjbN9SGiEjFpURERMqtL74Il1+eeQZycuDII8PttwccoJ5PRSoLJSIiUq5s3AgjR4YE5N13YbPNwpNvL74Ytt8+6ehEpLQpERGRcmHlynC55a674NtvoXVruPVWOOssaNw46ehEJFOUiIhIon74Ae65J9xyu3w57LUX3HgjHHMM1NAnlEilp7e5iJQ5d/joo3D55YUXQnuPY48Nt9/utVfS0YlIWVIiIiJlZsMGeP75kIB8+ik0agSXXQYXXhguxYhI1aNEREQybulSePhhuPde+PFH2G678P9pp0H9+klHJyJJUiIiIhkzfXpofDpkCKxZAwcfHPoD6d5d3a+LSKBERERKlTu8807o/XTkSKhVC048MbT/2GWXpKMTkfJGiYiIlIqffw4dj915J3z5Zehy/dpr4bzzYMstk45ORMorJSIi8rssWBC6Xr//fvjpp/DQuUGDQi1InTpJRyci5Z0SERHZJF99Fe5+eeopWL8ejjgidL/epYu6XxeR9CkREZG05ebCa6+FBOTtt6FuXTjzTLjkEmjfPunoRKQiUiIiIsVavRoefzzcATN9OrRsCTfdBP37Q5MmSUcnIhWZEhERKdTs2XDffaEPkKVLYc894emnQy+oNWsmHZ2IVAZKRETkf3z6abj88txz4Xbco48O7T/22UftP0SkdCkREREAcnLgxRfD7bcffggNG4a2HxdeCG3bJh2diFRWSkREqrjly8OTb++5JzwJd5ttQjJy+ukhGRERySQlIiJV1Lffwt13w+DBsGoV/OlPIQE58kioXj3p6ESkqlAiIlKFuMPYsaH9x4gRUKMG9O0bul/fffekoxORqkiJiEgVsH49PPtsSEAmTICmTeEvf4Hzz4ettko6OhGpypSIiFRiixaF7tfvuw/mz4eOHeGhh+Dkk6FevaSjExHZhETEzKoB9d19RQbiEZFSMGVKaO/xxBPhYXSHHQZDhkDXrrr9VkTKl2rpTGRmT5tZQzPbDJgETDGzKzIbmoiUhDv8978h6dhhh5CEnHIKTJ78a7mSEBEpb9JKRIDsWANyFPAa0BY4JWNRiUja1q4NPZ/usAMcfnh4GN0NN4ReUR9+GLKzk45QRKRw6V6aqWlmNQmJyL3uvsHMPINxiUgx5s4NbT8eeggWL4bddoOhQ+H446FWraSjExFJT7qJyEPATGAiMNbM2gBqIyKSgPHjQ/uPZ58NvaH26hW6X99/f116EZGKJ61ExN3vBu5OKfrBzA7KTEgikt/GjaHfjzvugPfeg/r14bzz4OKLYdttk45ORGTTpZWImFlT4O/AfoAD7wPXA4szF5qIrFgBjz0WekD97jto0wZuuw3OPBMaNUo6OhGR3y/dxqrDgIVAb+DY+P+zmQpKpKqbORMGDoSttw69nrZoEZ6EO2NGKFcSIiKVRbptRFq4+z9Thm8ws+MzEZBIVeUOH3wQ2n+8+CJUqwZ9+oREpHPnpKMTEcmMdBORN8ysL/CfOHws8HpmQhKpWjZsCLUdd9wB48bB5pvDFVfAhRdCq1ZJRyciklnpJiJnA5cCT8bhasBqMzsHcHfXw8JFSmjJktDPx733wpw50K4d3H8/nHoqbLZZ0tGJiJSNdO+aabApCzezOsBYoHZc13B3/7uZtSW0O2kKjAdOcff1m7IOkYpm2rRw+eXxx0NnZF26hL5ADj88XI4REalK0n7WjJn1BP4UB8e4+8g0ZlsHHOzuq2KHaO+b2WvAQOAOdx9mZg8CZwIPlDB2kQrDHd5+O1x+GTUKateGk04K7T922inp6EREkpPus2ZuBi4BpsTXJWZ2U3HzebAqDtaMLwcOBobH8scJPbaKVDo//wyDBsHOO8Ohh4Y2INddB7NmhXIlISJS1aVbI3IEsKu75wKY2ePABODPxc1oZtUJl1+2A+4DvgWWuXtOnORHoGUJ4xYp1xYsCO09HngAFi4Michjj0HfvlCnTtLRiYiUH2lfmgEaA0vi/2n3YuDuG4Fdzawx8CLQIZ35zKw/0B+gdevWJQhTJDkTJ4bLL888E+6G6dEjXH456CB1vy4iUpB0E5GbgAlmNhowQluRq0uyIndfFuffG2hsZjVirUgrYE4B0z8MPAzQqVMnPWBPyq3cXHj11ZCAjB4N9erB2WfDJZfA9tsnHZ2ISPmW7l0zz5jZGGDPWHSVu88vbj4zawZsiElIXeBQ4BZgNKEvkmHAacDLmxC7SKJWrYIhQ+Cuu0KPp61awS23hCRk882Tjk5EpGJIt7GqAV0I7URGALXMLJ2+HlsAo83sS+Az4M14t81VwEAzm0G4hXfQJkUvkoDZs+HKK0P36xddBE2bwrBh4VkwV16pJEREpCTSvTRzP5BLuNvlemAl8Dy/1pAUyN2/BHYroPw7QJ1WS4XzzDNw+umh/Ufv3jBgAOy9d9JRiYhUXOkmIn90993NbAKAuy81s1oZjEukXHGHG2+Ev/4V9t8fnngiPAlXRER+n3QTkQ3xNlyHX9p+5GYsKpFyZMMGOPdcGDw4dEI2aFDokExERH6/dDuUvptw6+0fzOxfwPuEO2lEKrVly0LX64MHw9/+FmpClISIiJSedO+aecrMxhMarBpwlLt/ndHIRBL2ww9wxBEwfXrojKxfv6QjEhGpfNJKRMzsTHcfBExNKbvZ3UvUl4hIRfHZZ3DkkaGL9tdfh4MPTjoiEZHKKd1LM73N7KS8ATO7D2iWmZBEkvXyy3DAAVC3Lnz0kZIQEZFMSrexam9ghJnlAt0Iz4o5M3NhiZQ999A52cCBsOeeMGIENG+edFQiIpVbkTUiZtbEzJoAdYGzgCsJfYj8I5aLVAobN8LFF4d+QY46KnTVriRERCTziqsRGU+4ZddS/naPLwe2yWh0ImVg1So44QQYORIuuyx00169etJRiYhUDUUmIu7etqwCEUnC3LnhCbkTJ8L998N55yUdkYhI1ZJuGxGRSuerr8LtuUuXwiuvhP9FRKRspXvXjEil8vrrsO++kJsL77+vJEREJCnFNVatWVaBiJSVhx+G7t2hbVv45BPYddekIxIRqbqKqxH5yMxeMrNzzSyrDOIRyZjcXLj6ajjnHDj00FAT0qpV0lGJiFRtxTVW7RQTkG7AnWbWkvCcmdeAd919XcYjFCkFa9fCaafBc8+FB9jdcw/UUAspEZHEFdtGxN1nuvuD7n4UsA/wCnAI8J6ZvZrpAEV+r4ULoUuXkITcemu4O0ZJiIhI+VCij2N33wC8E1/EGhKRcmvatNAQde5cGD4cevdOOiIREUn1u34Xuvuc0gpEpLSNHRt6Sa1RI/SUutdeSUckIiL56fZdqZSeeio0SG3eHD7+WEmIiEh5pUREKhV3uP56OPlk2Gcf+PBD2EYPIhARKbfSujRjZu2AK4A2qfO4ux6QLuXG+vXQvz88/jiccgo8+ijUqpV0VCIiUpR024g8BzwIPAJszFw4Iptm6dLQEHX0aLjuOrj2WjBLOioRESlOuolIjrs/kNFIRDbR99+HO2O+/RaeeCJclhERkYqhyETEzJrEf18xs/OBF4FfOjFz9yUZjE2kWJ98Aj17woYN8OabcMABSUckIiIlUVyNyHjAgbxK7itSxjmgZoCSmBdegJNOgq22gldfhQ4dko5IRERKqrgu3tuWVSAi6XKH22+HK66AP/4RRoyAZs2SjkpERDZFWrfvmtkFZtY4ZXjzeKlGpEzl5MAFF8Dll4fGqe+8oyRERKQiS7cfkbPdfVnegLsvBc7OTEgiBVu5Enr1ggcegCuvhGefhbp1k45KRER+j3TvmqluZubuDmBm1QH10CBlZs4c6NEDvvoKHnoo9BciIiIVX7qJyOvAs2b2UBw+B/hvZkIS+a2JE6F7d1ixAkaOhG7dko5IRERKS7qJyJVAf+C8OPwm8GhGIhJJ8dprcNxx0LgxvP8+7Lxz0hGJiEhpKjYRiZdhhrr7SYTeVUXKxIMPwoUXhuTjlVegZcukIxIRkdJWbGNVd98ItDEztQmRMpGbG27NPe+8cBlm7FglISIilVW6l2a+Az4wsxHA6rxCd789I1FJlbV2bXhg3fPPh9t077wTaqR7loqISIWT7kf8t/FVDWiQuXCkKluwINye++mnocOySy/Vg+tERCq7tBIRd/8HgJnVj8OrMhmUVD1ffx0eXLdgQei6/aijko5IRETKQro9q+5oZhOAycBkMxtvZjsUM8/WZjbazKaY2WQzuySWNzGzN83sm/h389+/GVKRjR4N++wDa9bAu+8qCRERqUrS7Vn1YWCgu7dx9zbAZcAjxcyTA1zm7tnAXsAFZpYNXA287e7bA2/HYamihg6Fww4LD6775BPYc8+kIxIRkbKUbiKymbuPzhtw9zHAZkXN4O7z3P3z+P9K4GugJdALeDxO9jig3z/OsREAAB1DSURBVL9VkDtcdx2cdhrsvz988AFkZSUdlYiIlLW075oxs78BT8Thkwl30qTFzLKA3YBPgObuPi+Omg80L2Se/oRO1GjdunW6q5IKYN06OOssePJJ6NcvdNleSzeHi4hUSenWiJwBNANeAJ4HtgBOT2fG2MD1eeBSd1+ROi4+u8YLms/dH3b3Tu7eqZker1ppLFkCXbuGJOSGG2DwYCUhIiJVWbo1Ioe4+8WpBWbWB3iuqJnMrCYhCXnK3V+IxQvMrIW7zzOzFsBPJQ1aKqZvvw3PjPn+e3j6aTjhhKQjEhGRpKVbI/LnNMt+YWYGDAK+ztfx2QjgtPj/acDLacYgFdhHH8Fee8HChfDWW0pCREQkKLJGxMwOB44AWprZ3SmjGhLuiinKvsApwFdm9kUs+wtwM/AfMzsT+AE4blMCl4rjuefg1FNDN+2jRkG7dklHJCIi5UVxl2bmAuOAnsD4lPKVwICiZnT394HC+sXskm6AUnG5w623wlVXhX5CXn4Zttgi6ahERKQ8KTIRcfeJwEQzWwqMdPfcsglLKrqcnPCsmIcfhuOPhyFDoE6dpKMSEZHyJt02IscB35jZ/5lZh0wGJBXfihXQo0dIQv7859AwVUmIiIgUJN1nzZxsZg2BE4AhZubAY8AzsbMyEQBmzw53xkyZAo88EvoLERERKUy6NSLEPkCGA8OAFsDRwOdmdlGGYpMKZsKEcGfMDz/Aa68pCRERkeKl+9C7nmb2IjAGqAl0dvfDgV0Iz52RKm7kyNBVe/Xqobv2Qw9NOiIREakI0u3QrDdwh7uPTS109zXxNlypwu67Dy6+GHbbDV55BVq0SDoiERGpKNKqEXH304DpsWbkSDPbMmXc2xmLTsq1jRth4EC48MLQLuTdd5WEiIhIyaR7aeZM4FPgGOBY4GMzOyOTgUn5tmYNHHss3HFHqA158UXYrMjnMYuIiPyvdC/NXAns5u6LAcysKfAhMDhTgUn5NX8+9OwJ48fDXXeFRERERGRTpJuILCb0pppnZSyTKmby5HAZZuHCUAvSs2fSEYmISEWWbiIyA/jEzF4GHOgFfGlmAwHyPdROKqm334bevaFuXRg7FvbYI+mIRESkoks3Efk2vvLkPTG3QemGI+XVY49B//7Qvn14cF3r1klHJCIilUG6Pav+I9OBSPnkDtdeCzfcEPoGee45aNQo6ahERKSySLdGRKqgdevgjDPCs2LOOgvuvx9q1kw6KhERqUyUiEiBFi+Go46C99+Hm26Cq64Cs6SjEhGRykaJiPyPGTPgiCNg1iwYNgyOPz7piEREpLJKt0Ozdmb2tplNisM7m9lfMxuaJOGDD8KD65YsCXfJKAkREZFMSvfpu48AfwY2ALj7l0DfTAUlyXj2WejSBZo0gY8/hn33TToiERGp7NJNROq5+6f5ynJKOxhJhntoB9K3L3TuDB99BNttl3RUIiJSFaTbRmSRmW1L6MwMMzsWmJexqKTMbNgA550HgwbBiSfC4MFQu3bSUYmISFWRbiJyAfAw0MHM5gDfAydnLCopE8uXQ58+8Oab8Ne/wvXX684YEREpW+l2aPYdcIiZbQZUc/eVxc0j5dusWeGZMVOnhlqQ009POiIREamK0kpEzKwxcCqQBdSw+LPZ3fXc1Qpo/Hjo0QPWroX//jc0UBUREUlCupdmRgEfA18BuZkLRzJtxAg44QRo1izcnpudnXREIiJSlaWbiNRx94EZjUQy7u674dJLoVOnkJBsuWXSEYmISFWX7u27T5jZ2WbWwsya5L0yGpmUmo0b4ZJLwqtXLxgzRkmIiIiUD+nWiKwHbgWuId7CG/9uk4mgpPSsXh1uyx0xAgYOhP/7P6hePemoREREgnQTkcuA7dx9USaDkdI1bx4ceSRMmAD33gsXXJB0RCIiIr+VbiIyA1iTyUCkdE2aFG7PXbw41IZ07550RCIiIv8r3URkNfCFmY0G1uUV6vbd8umNN0JHZfXrw3vvwW67JR2RiIhIwdJNRF6KLynnHn0Uzj0XdtgBXn0VWrVKOiIREZHCpduz6uOZDkR+n9zc0E37TTdBt27hSboNGyYdlYiISNGKTETM7D/ufpyZfcWvd8v8wt13zlhkkraff4Z+/ULycc45oWFqjXTrukRERBJU3NfVJfFvj0wHIptm0aLQN8iHH4Zbcy+/XA+uExGRiqPIDs3cfV7893x3/yH1BZyf+fCkKD/8AHvvDZ9/Ds89B1dcoSREREQqlnR7Vj20gLLDSzMQKZkFC+CQQ0KNyDvvwLHHJh2RiIhIyRWZiJjZebF9SHsz+zLl9T3wZXELN7PBZvaTmU1KKWtiZm+a2Tfx7+a/fzOqlmXL4LDDYO5cGDUq1IqIiIhURMXViDwNHAmMiH/zXnu4+8lpLH8I0C1f2dXA2+6+PfB2HJY0rV4dOif7+mt46SUlISIiUrEV2VjV3ZcDy4ETNmXh7j7WzLLyFfcCDoz/Pw6MAa7alOVXNevXQ+/e8PHHoU3IoQVdMBMREalA0m0jUpqapzSCnQ80L2giM+tvZuPMbNzChQvLLrpyauNGOPlkeP310GnZMcckHZGIiMjvl0Qi8gt3dwronySOe9jdO7l7p2bNmpVxZOWLe+gf5Lnn4Pbb4fTTk45IRESkdCSRiCwwsxYA8e9PCcRQYbjDlVfCoEHwt7/BgAFJRyQiIlJ6kkhERgCnxf9PA15OIIYK46ab4N//hosugn/8I+loRERESldGExEzewb4iHD7749mdiZwM3ComX0DHBKHpQAPPADXXAOnnAJ33qnOykREpPLJ6BNJ3L2wu226ZHK9lcHTT8MFF0DPnuGyTLVEW/OIiIhkhr7eyqGRI+HUU+HAA8OD7GrWTDoiERGRzFAiUs68+y706QO77w4vvwx16iQdkYiISOYoESlHxo2DI4+EbbaB116DBg2SjkhERCSzlIiUE19/Dd26QdOm8MYb4a+IiEhlp0SkHJg5M3TXXrMmvPUWtGyZdEQiIiJlI6N3zUjx5s8PSciaNaF9yLbbJh2RiIhI2VEikqClS6FrV5g3L9SE7LRT0hGJiIiULSUiCVm9Grp3h2nT4NVXYa+9ko5IRESk7CkRScC6dXD00fDJJ+FBdoccknREIiIiyVAiUsZycuCkk+DNN+Gxx+CYY5KOSEQks8aPH/+HGjVqPArsiG6SqGpygUk5OTln7bHHHgU+5FaJSBlyh3POgeefhzvugH79ko5IRCTzatSo8eiWW27ZsVmzZkurVavmSccjZSc3N9cWLlyYPX/+/EeBngVNo8y0jLjD5ZfD4MFw7bVw6aVJRyQiUmZ2bNas2QolIVVPtWrVvFmzZssJtWEFT1OG8VRp//oX3H47XHQRXHdd0tGIiJSpakpCqq547AvNN5SIlIF774W//S08yO7OO8Es6YhERETKByUiGfbkk6EWpFcvGDQIqmmPi4iUe1dfffWWpbWsgQMHbnXttdc2L2z8iSee2PqNN97YrHPnzu3Hjh1br7TWW5pGjhzZoEGDBrt26NAhu0OHDtmXX355i7xxw4cPb5iVlbVj69atd/zLX/5S4v2mr8UMGjEiNEg9+GAYNgxqqGmwiEiFcPfdd7cofqpf5ebmsnHjxk1a1+eff17/4IMPXr1JM5ehTp06rZo6deqUqVOnTvn3v/89DyAnJ4cBAwa0HjVq1PTp06dPfv7555uMHz++RM+NVyKSIaNHw3HHwR57wEsvQZ0SHRYRESkt06ZNq9W2bdsdevbs2XabbbbZoVu3btusXLmy2ogRIxoccsghvzxY48UXX2x46KGHbnv++ee3XLduXbUOHTpk9+zZsy3Adddd13z77bffYfvtt9/h+uuv/0PecrOysnY8+uijs9q1a7fDt99+W2v48OENs7OzO7Zv3z577733bpe37K+//rpu586d27dq1WqnG2644Q955Z9//nmdbbbZ5ucaKb9UN27cSO/evbMuvvjirQBeeOGFhrvuumuH7Ozsjocffvg2y5cvrwbw8ssvN+jYsWN2u3btsvv06ZO1du1aA2jZsuVO5557bqt27dpl77TTTh0nTZpUG2Dw4MGbb7/99ju0b98+u1OnTu1LY9+OGTNmszZt2qzLzs5eX6dOHT/mmGOWDB8+vHFJlqHf6Bnw2WfQsydstx2MGgUNGiQdkYhI+XDGGWw9aRKlevlhxx1ZM3gws4uaZubMmXUeeuihmV27dl3dp0+frFtvvbXZddddt+CSSy5pPXfu3BpbbbVVzuDBg5uefvrpi0488cTlQ4YM+cPUqVOnALz33nv1nn766abjx4//2t3ZY489Onbp0mXlFltssXHWrFm1Bw0a9H2XLl1mzp07t8aFF16YNWbMmKkdOnRYv2DBgup5658xY0adDz/8cNqyZcuqd+zYcccrrrhiYe3atX3EiBGNunbtujxvug0bNthRRx3VNjs7e+0tt9wyf968eTVuvPHGFmPHjp3esGHD3GuuuWbLf/7zn82vv/76+eecc07bN954Y9rOO++87uijj8669dZbm1177bU/ATRq1Chn+vTpU+69996mF1100dajR4+ecfPNN7d44403prdt23bDokWLqv/vXirahAkT6rdv3z67efPmG26//fbZnTp1+nn27Nm1WrZsuT5vmlatWq3/5JNP6pdkuaoRKWVTpkC3btCsGbzxBjRtmnREIiKy5ZZbru/atetqgFNOOWXxhx9+WL9atWocd9xxix955JEmixYtqv7555/X79Onz/L8844ZM6b+EUccsaxhw4a5jRo1yu3evfvS0aNHNwBo0aLF+i5duqyO023WuXPnlR06dFgP0Lx581+u1XTt2nVZ3bp1vUWLFjlNmjTZ8OOPP9YAeOuttxoeddRRK/KmO//889vkJSF5y/z222/rdO7cuUOHDh2yhw0b1nTWrFm1Jk6cWKdVq1brdt5553UA/fr1W/z+++//8rP3tNNOWwJw9tlnL5kwYUJ9CJdWTjrppKzbbrtti5ycnBLtv3322Wf1Dz/88OW0adOmXHDBBT/17t17uxItoAiqESlF338fnqRbq1boOXWrrZKOSESkfCmu5iJTLN/tinnD55133uLu3btvV6dOHT/yyCOX1qxZs0TLrVevXm4609WuXfuX25erV69OTk6OrVy5stqKFSuqZ2Vlbcgb16lTp1XvvfdewzVr1iyoV6+euzv77bffildeeeX71OV99NFHdYtaX7WUOyPMzAGefvrpWe+8885mI0aMaLTHHntkjx8/fsqWW275S7J00UUXtXzzzTcbAeTVBuVp0qTJL9t5/PHHLx84cGDrefPm1dh6663Xz5kzp1beuB9//PE3NSTpUI1IKZk3LyQha9eGJGTbbYufR0REysa8efNqvfXWW5sBPPXUU0322WefVQBZWVkbmjdvvuG2225r0b9//0V509eoUcPXrVtnAAcddNCqUaNGNY6JQ7VRo0ZtftBBB63Mv44DDzxw9aefftpg6tSptQBSL80U5NVXX22w3377/WY555xzzqKuXbsu79Gjx7YbNmzgwAMPXD1u3Lj6ee08VqxYUe3LL7+svcsuu/w8Z86cWnnlQ4cObbr//vv/sqyhQ4c2ARg0aNDmu+2222qAyZMn1z744INX33nnnXM333zznO+++65W6rrvueeeOXmNUfPHOmvWrBq5uSEXGT16dL3c3FyaN2+ec8ABB6yeOXNmnalTp9b6+eef7YUXXmjSu3fvZUVtd36qESkFS5bAYYfB/Pnw9tuwY6H9x4mISBKysrJ+vueee/7Qv3//ettvv/3Pl19++cK8cX379l1833331dh9991/zis76aSTFnbs2DF7xx13XDNixIjvTzzxxMW77757R4BTTjll4b777rt22rRpv/ki32qrrXLuvvvumUcfffR2ubm5NG3adMOHH374TWExjRo1qtFxxx23NH/5ddddt2DAgAHVjznmmLYvvfTS9w899NDMvn37brN+/XoD+Pvf/z5n5513Xvfggw/O7NOnz7YbN25kl112WZO6TUuXLq3erl277Fq1avmwYcO+AxgwYECrmTNn1nZ322+//Vbstddea9Pdf08++eTmgwcP/kP16tW9Tp06uUOHDv2uWrVqVKtWjdtuu21Wt27d2m3cuJETTzxxUadOnX4ufom/Mvfy39ldp06dfNy4cUmHUaCVK6FrV/j889AwtUuXpCMSEQnMbLy7d0o6jokTJ87cZZddFhU/ZWZMmzatVo8ePbb/5ptvJhc0/tRTT2292267rRkwYECZxpidnd1xwoQJU1Mv25SGli1b7jRu3LivW7RoUbKGIBk0ceLELXbZZZesgsapRuR3mDULjjwSJk+G555TEiIiUtHssMMOHevWrZv70EMPlXnblSlTpnxd1ussj5SIbKJPPw236K5dG2pCunZNOiIRESlI+/bt1xdWGzJ58uRKlwzMmTPnq6RjKAk1Vt0Ew4fDAQdAvXrw0UdKQkREipGbm5urp2xVUfHYF3p3kRKREnCHG2+EPn1g993hk08gOzvpqEREyr1JCxcubKRkpOrJzc21hQsXNgImFTaNLs2kad066N8fhg6Fk06CRx9Vt+0iIunIyck5a/78+Y/Onz9/R/QDuKrJBSbl5OScVdgESkTSsGgRHH00vP8+XH89/PWvYMrrRUTSsscee/wE9Ew6DimflIgUY+pU6N4d5swJT9A9/vikIxIREak8lIgU4a234NhjoXZtGDMG9tor6YhEREQqF12rK8RDD4WH1229dbhVV0mIiIhI6VMiks/GjTBwIJx7brgt94MPoE2bpKMSERGpnJSIpFi1KjRKveMOuPhiGDECGjZMOioREZHKS21EotmzQ3ftkybBfffB+ecnHZGIiEjlV+UTkdxcePddOPFEWLMGXn01PElXREREMi+xSzNm1s3MppnZDDO7uqzW6w7Tp8MDD4Q7Ypo1g4MPDp2TffihkhAREZGylEiNiJlVB+4DDgV+BD4zsxHuPiUT65szB95+G955J/z98cdQvvXW4cF1XbpAjx7QuHEm1i4iIiKFSerSTGdghrt/B2Bmw4BeQKkmIl99FZ4LM21aGG7aFA46KCQeXbrAdtuph1QREZEkJZWItARmpwz/CPwxdQIz6w/0B2jduvUmrWTrrWHbbeHss0PisfPOUE33CYmIiJQb5baxqrs/DDwM0KlTJ9+UZTRuHBqfioiISPmUVP3AHGDrlOFWsUxERESqkKQSkc+A7c2srZnVAvoCIxKKRURERBKSyKUZd88xswuB14HqwGB3n5xELCIiIpKcxNqIuPsoYFRS6xcREZHk6R4SERERSYwSEREREUmMEhERERFJjBIRERERSYy5b1JfYWXKzBYCPxQz2RbAojIIp7ypqtsNVXfbtd1Vy+/Z7jbu3qw0gxEpbRUiEUmHmY1z905Jx1HWqup2Q9Xddm131VJVt1uqDl2aERERkcQoEREREZHEVKZE5OGkA0hIVd1uqLrbru2uWqrqdksVUWnaiIiIiEjFU5lqRERERKSCUSIiIiIiiakUiYiZdTOzaWY2w8yuTjqeTWFmW5vZaDObYmaTzeySWN7EzN40s2/i381juZnZ3XGbvzSz3VOWdVqc/hszOy2lfA8z+yrOc7eZWdlv6f8ys+pmNsHMRsbhtmb2SYzzWTOrFctrx+EZcXxWyjL+HMunmdlhKeXl9twws8ZmNtzMpprZ12a2dxU53gPiOT7JzJ4xszqV8Zib2WAz+8nMJqWUZfz4FrYOkXLL3Sv0C6gOfAtsA9QCJgLZSce1CdvRAtg9/t8AmA5kA/8HXB3LrwZuif8fAbwGGLAX8EksbwJ8F/9uHv/fPI77NE5rcd7Dk97uGNdA4GlgZBz+D9A3/v8gcF78/3zgwfh/X+DZ+H92PO61gbbxfKhe3s8N4HHgrPh/LaBxZT/eQEvge6BuyrHuVxmPOfAnYHdgUkpZxo9vYevQS6/y+qoMNSKdgRnu/p27rweGAb0SjqnE3H2eu38e/18JfE340O5F+MIi/j0q/t8LGOrBx0BjM2sBHAa86e5L3H0p8CbQLY5r6O4fu7sDQ1OWlRgzawV0Bx6NwwYcDAyPk+Tf5rx9MRzoEqfvBQxz93Xu/j0wg3BelNtzw8waEb6oBgG4+3p3X0YlP95RDaCumdUA6gHzqITH3N3HAkvyFZfF8S1sHSLlUmVIRFoCs1OGf4xlFVasft4N+ARo7u7z4qj5QPP4f2HbXVT5jwWUJ+1O4EogNw43BZa5e04cTo3zl22L45fH6Uu6L8qDtsBC4LF4WepRM9uMSn683X0O8G9gFiEBWQ6Mp2occyib41vYOkTKpcqQiFQqZlYfeB641N1XpI6Lv3wqzf3WZtYD+MndxycdSwJqEKrtH3D33YDVhGr0X1S24w0Q2yv0IiRiWwGbAd0SDSohZXF8K+M5JJVPZUhE5gBbpwy3imUVjpnVJCQhT7n7C7F4QayGJf79KZYXtt1FlbcqoDxJ+wI9zWwmoQr9YOAuQrV0jThNapy/bFsc3whYTMn3RXnwI/Cju38Sh4cTEpPKfLwBDgG+d/eF7r4BeIFwHlSFYw5lc3wLW4dIuVQZEpHPgO1jq/tahAZtIxKOqcTide9BwNfufnvKqBFAXkv504CXU8pPja3t9wKWx+rY14GuZrZ5/PXZFXg9jlthZnvFdZ2asqxEuPuf3b2Vu2cRjts77n4SMBo4Nk6Wf5vz9sWxcXqP5X3jHRZtge0JDfnK7bnh7vOB2WbWPhZ1AaZQiY93NAvYy8zqxbjytrvSH/OoLI5vYesQKZ+Sbi1bGi9Ci/PphNby1yQdzyZuw36EKtQvgS/i6wjC9fC3gW+At4AmcXoD7ovb/BXQKWVZZxAa780ATk8p7wRMivPcS+xZtzy8gAP59a6ZbQhfKjOA54DasbxOHJ4Rx2+TMv81cbumkXJ3SHk+N4BdgXHxmL9EuCui0h9v4B/A1BjbE4Q7XyrdMQeeIbSD2UCoATuzLI5vYevQS6/y+lIX7yIiIpKYynBpRkRERCooJSIiIiKSGCUiIiIikhglIiIiIpIYJSIiIiKSGCUiUqmZ2UYz+8LMJprZ52a2TzHTNzaz89NY7hgz67SJMY0ys8abMq+ISGWjREQqu7Xuvqu77wL8GbipmOkbE574mjHufoSHB9yJiFR5SkSkKmkILIXwTB8zezvWknxlZnlPaL0Z2DbWotwap70qTjPRzG5OWV4fM/vUzKab2f75V2ZmLcxsbFzWpLxpzGymmW1hZufGcV+Y2fdmNjqO72pmH8XYnovPHxIRqZTUoZlUama2kdBTZR2gBXCwu4/PewS9u68wsy2AjwndhLch9PC6Y5z/cOBvwCHuvsbMmrj7EjMbA4x398vM7AhgoLsfkm/dlwF13P1fZlY9rm9lfLZOJ3dfFKerCbwD/B/wEeH5K4e7+2ozu4rQy+j1mdxPIiJJqVH8JCIV2lp33xXAzPYGhprZjoQutW80sz8BuYRHqBf0uPRDgMfcfQ2Auy9JGZf3YMLxQFYB834GDI6Jxkvu/kUhMd5FeIbKK/GJxNnAB+ERItQiJCciIpWSEhGpMtz9o1j70YzwPJJmwB7uviHWUtQp4SLXxb8bKeC95O5jY6LTHRhiZre7+9DUacysH6EW5sK8IuBNdz+hhLGIiFRIaiMiVYaZdQCqEx4j3wj4KSYhBxGSAYCVQIOU2d4ETjezenEZTUqwvjbAAnd/BHgU2D3f+D2Ay4GT3T03Fn8M7Gtm28VpNjOzdiXbUhGRikM1IlLZ1TWzvEsiBpzm7hvN7CngFTP7ivAE3KkA7r7YzD4ws0nAa+5+hZntCowzs/XAKOAvaa77QOAKM9sArCI8qj3VhUATYHS8DDPO3c+KtSTPmFntON1fCU+TFRGpdNRYVURERBKjSzMiIiKSGCUiIiIikhglIiIiIpIYJSIiIiKSGCUiIiIikhglIiIiIpIYJSIiIiKSmP8Hrauy2dHrJQoAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plot_relative_time(experiments_eq, results, [50], max_batch_size=max_batch_size)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, "source": [ "## Conclusion\n", "\n", - "As illustrated in the experiments, KeOps allows you to drastically speed up and scale up drift detection to larger datasets without running into memory issues. The speed benefit of KeOps over the PyTorch (or TensorFlow) MMD detector decreases as the number of features increases. Note though that it is not advised to apply the (untrained) MMD detector to very high-dimensional data in the first place." + "As illustrated in the experiments, KeOps allows you to drastically speed up and scale up drift detection to larger datasets without running into memory issues. The speed benefit of KeOps over the PyTorch (or TensorFlow) MMD detectors decrease as the number of features increases. Note though that it is not advised to apply the (untrained) MMD detector to very high-dimensional data in the first place and that we can apply dimensionality reduction via the deep kernel for the learned kernel MMD detector." ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python [conda env:detect] *", "language": "python", - "name": "python3" + "name": "conda-env-detect-py" }, "language_info": { "codemirror_mode": { @@ -588,9 +630,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.7.6" } }, "nbformat": 4, "nbformat_minor": 5 -} \ No newline at end of file +} From faca2349331bdb1e3eeb4fe331c1e2013eda1e14 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Fri, 9 Sep 2022 14:35:02 +0100 Subject: [PATCH 09/35] Ignore tests in codecov (#614) --- codecov.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/codecov.yml b/codecov.yml index ca612fcc6..12d264eac 100644 --- a/codecov.yml +++ b/codecov.yml @@ -23,3 +23,7 @@ comment: layout: "reach,diff,flags,tree" behavior: default require_changes: no + +ignore: + - "**/tests/*" # ignore anything in tests/ directories + - "**/test_*.py" # ignore test_*.py files even if they are located elsewhere From 0d46a04d2db7a9e737b096cf6c2ab0330819f018 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Fri, 9 Sep 2022 17:52:01 +0100 Subject: [PATCH 10/35] Add some missing logic for failed URI's in datasets and test_saving (#607) --- CHANGELOG.md | 1 + alibi_detect/datasets.py | 7 ++++++- alibi_detect/saving/tests/datasets.py | 7 ++++++- alibi_detect/saving/tests/test_saving.py | 16 +++++++++++++--- alibi_detect/tests/test_datasets.py | 11 +++++++++-- 5 files changed, 35 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d913dd9b0..0f4b7fef9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ See the [documentation](https://docs.seldon.io/projects/alibi-detect/en/latest/c ### Development - UTF-8 decoding is enforced when `README.md` is opened by `setup.py`. This is to prevent pip install errors on systems with `PYTHONIOENCODING` set to use other encoders ([#605](https://github.com/SeldonIO/alibi-detect/pull/605)). +- Skip specific save/load tests that require downloading remote artefacts if the relevant URI(s) is/are down ([#607](https://github.com/SeldonIO/alibi-detect/pull/607)). ## v0.10.3 ## [v0.10.3](https://github.com/SeldonIO/alibi-detect/tree/v0.10.3) (2022-08-17) diff --git a/alibi_detect/datasets.py b/alibi_detect/datasets.py index 507a25ba7..f3015e06a 100644 --- a/alibi_detect/datasets.py +++ b/alibi_detect/datasets.py @@ -11,6 +11,7 @@ from alibi_detect.utils.data import Bunch from alibi_detect.utils.url import _join_url from requests import RequestException +from urllib.error import URLError from scipy.io import arff from sklearn.datasets import fetch_kddcup99 @@ -59,7 +60,11 @@ def fetch_kdd(target: list = ['dos', 'r2l', 'u2r', 'probe'], """ # fetch raw data - data_raw = fetch_kddcup99(subset=None, data_home=None, percent10=percent10) + try: + data_raw = fetch_kddcup99(subset=None, data_home=None, percent10=percent10) + except URLError: + logger.exception("Could not connect, URL may be out of service") + raise # specify columns cols = ['duration', 'protocol_type', 'service', 'flag', 'src_bytes', 'dst_bytes', diff --git a/alibi_detect/saving/tests/datasets.py b/alibi_detect/saving/tests/datasets.py index 6f9cb607b..eadb83925 100644 --- a/alibi_detect/saving/tests/datasets.py +++ b/alibi_detect/saving/tests/datasets.py @@ -1,6 +1,8 @@ import numpy as np +import pytest from alibi_testing.data import get_movie_sentiment_data from pytest_cases import parametrize +from requests import RequestException # Note: If any of below cases become large, see https://smarie.github.io/python-pytest-cases/#c-caching-cases FLOAT = np.float32 @@ -63,4 +65,7 @@ def data_synthetic_nd(data_shape): class TextData: @staticmethod def movie_sentiment_data(): - return get_movie_sentiment_data() + try: + return get_movie_sentiment_data() + except RequestException: + pytest.skip('Movie sentiment dataset URL down') diff --git a/alibi_detect/saving/tests/test_saving.py b/alibi_detect/saving/tests/test_saving.py index e40a71e10..badf7c1c3 100644 --- a/alibi_detect/saving/tests/test_saving.py +++ b/alibi_detect/saving/tests/test_saving.py @@ -9,6 +9,7 @@ from functools import partial from pathlib import Path from typing import Callable +from requests.exceptions import HTTPError import toml import dill @@ -202,7 +203,10 @@ def nlp_embedding_and_tokenizer(model_name, max_len, uae, backend): backend = 'tf' if backend == 'tensorflow' else 'pt' # Load tokenizer - tokenizer = AutoTokenizer.from_pretrained(model_name) + try: + tokenizer = AutoTokenizer.from_pretrained(model_name + 'TODO') + except (OSError, HTTPError): + pytest.skip(f"Problem downloading {model_name} from huggingface.co") X = 'A dummy string' # this will be padded to max_len tokens = tokenizer(list(X[:5]), pad_to_max_length=True, max_length=max_len, return_tensors=backend) @@ -214,13 +218,19 @@ def nlp_embedding_and_tokenizer(model_name, max_len, uae, backend): enc_dim = 32 if backend == 'tf': - embedding = TransformerEmbedding_tf(model_name, emb_type, layers) + try: + embedding = TransformerEmbedding_tf(model_name, emb_type, layers) + except (OSError, HTTPError): + pytest.skip(f"Problem downloading {model_name} from huggingface.co") if uae: x_emb = embedding(tokens) shape = (x_emb.shape[1],) embedding = UAE_tf(input_layer=embedding, shape=shape, enc_dim=enc_dim) else: - embedding = TransformerEmbedding_pt(model_name, emb_type, layers) + try: + embedding = TransformerEmbedding_pt(model_name, emb_type, layers) + except (OSError, HTTPError): + pytest.skip(f"Problem downloading {model_name} from huggingface.co") if uae: x_emb = embedding(tokens) emb_dim = x_emb.shape[1] diff --git a/alibi_detect/tests/test_datasets.py b/alibi_detect/tests/test_datasets.py index fbe324e19..26b02c6a9 100644 --- a/alibi_detect/tests/test_datasets.py +++ b/alibi_detect/tests/test_datasets.py @@ -2,6 +2,7 @@ import pandas as pd import pytest from requests import RequestException +from urllib.error import URLError from alibi_detect.datasets import fetch_kdd, fetch_ecg, corruption_types_cifar10c, fetch_cifar10c, \ fetch_attack, fetch_nab, get_list_nab from alibi_detect.utils.data import Bunch @@ -24,7 +25,7 @@ def test_fetch_kdd(return_X_y): keep_cols = np.random.choice(keep_cols_list, 5, replace=False) try: data = fetch_kdd(target=target, keep_cols=keep_cols, percent10=True, return_X_y=return_X_y) - except RequestException: + except URLError: pytest.skip('KDD dataset URL down') if return_X_y: assert isinstance(data, tuple) @@ -53,13 +54,19 @@ def test_fetch_ecg(return_X_y): # CIFAR-10-C dataset -corruption_list = corruption_types_cifar10c() +try: + corruption_list = corruption_types_cifar10c() +except RequestException: + corruption_list = None +@pytest.mark.skipif(corruption_list is None, reason="CIFAR-10-C dataset URL is down") def test_types_cifar10c(): + print(corruption_list) assert len(corruption_list) == 19 +@pytest.mark.skipif(corruption_list is None, reason="CIFAR-10-C dataset URL is down") @pytest.mark.parametrize('return_X_y', [True, False]) def test_fetch_cifar10c(return_X_y): corruption = list(np.random.choice(corruption_list, 5, replace=False)) From f1a63fc902d98d5126de334202d9ce872bdd94de Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Thu, 15 Sep 2022 10:24:47 +0100 Subject: [PATCH 11/35] Correct platform-specific installs in CI, and add codecov tags (#615) --- .github/workflows/ci.yml | 6 +++--- .github/workflows/test_all_notebooks.yml | 2 +- .github/workflows/test_changed_notebooks.yml | 2 +- setup.cfg | 1 - 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05707f30a..b5c85b36f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,10 +51,10 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install --upgrade --upgrade-strategy eager -r requirements/dev.txt python -m pip install --upgrade --upgrade-strategy eager -e . - if [ "$RUNNER_OS" != "Windows" ] && [ ${{ matrix.python }} < '3.10' ]; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. + if [ "$RUNNER_OS" != "Windows" ] && ${{ matrix.python-version != '3.10' }}; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. python -m pip install --upgrade --upgrade-strategy eager -e .[prophet] fi - if [ "$RUNNER_OS" == "Linux"]; then # Currently, we only support KeOps on Linux. + if [ "$RUNNER_OS" == "Linux" ]; then # Currently, we only support KeOps on Linux. python -m pip install --upgrade --upgrade-strategy eager -e .[keops] fi python -m pip install --upgrade --upgrade-strategy eager -e .[tensorflow,torch] @@ -75,7 +75,7 @@ jobs: - name: Upload coverage to Codecov if: ${{ success() }} run: | - codecov + codecov -F ${{ matrix.os }}-${{ matrix.python-version }} - name: Build Python package run: | diff --git a/.github/workflows/test_all_notebooks.yml b/.github/workflows/test_all_notebooks.yml index 73b0026b6..5526a51f7 100644 --- a/.github/workflows/test_all_notebooks.yml +++ b/.github/workflows/test_all_notebooks.yml @@ -41,7 +41,7 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install --upgrade --upgrade-strategy eager -r requirements/dev.txt -r testing/requirements.txt python -m pip install --upgrade --upgrade-strategy eager -e . - if [ "$RUNNER_OS" != "Windows" ] && [ ${{ matrix.python }} < '3.10' ]; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. + if [ "$RUNNER_OS" != "Windows" ] && ${{ matrix.python-version != '3.10' }}; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. python -m pip install --upgrade --upgrade-strategy eager -e .[prophet] fi python -m pip install --upgrade --upgrade-strategy eager -e .[torch,tensorflow] diff --git a/.github/workflows/test_changed_notebooks.yml b/.github/workflows/test_changed_notebooks.yml index fa275da43..d32e1dea0 100644 --- a/.github/workflows/test_changed_notebooks.yml +++ b/.github/workflows/test_changed_notebooks.yml @@ -56,7 +56,7 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install --upgrade --upgrade-strategy eager -r requirements/dev.txt -r testing/requirements.txt python -m pip install --upgrade --upgrade-strategy eager -e . - if [ "$RUNNER_OS" != "Windows" ] && [ ${{ matrix.python }} < '3.10' ]; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. + if [ "$RUNNER_OS" != "Windows" ] && ${{ matrix.python-version != '3.10' }}; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. python -m pip install --upgrade --upgrade-strategy eager -e .[prophet] fi python -m pip install --upgrade --upgrade-strategy eager -e .[torch,tensorflow] diff --git a/setup.cfg b/setup.cfg index 5b4de7f3c..926613e4d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,7 +11,6 @@ addopts = --tb native -W ignore --cov=alibi_detect - --cov-append --randomly-dont-reorganize --randomly-seed=0 #-n auto From 3c859776abebe6cc07c1f3870392213c3193e6b6 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Thu, 15 Sep 2022 12:02:35 +0100 Subject: [PATCH 12/35] Temporary fix for mypy issue in Python 3.10 CI (#619) --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5c85b36f..82ada6464 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,12 +30,12 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: [ '3.7', '3.8', '3.9', '3.10' ] + python-version: [ '3.7', '3.8', '3.9', '3.10.6' ] include: # Run macos and windows tests on only one python version - os: windows-latest python-version: '3.9' # PyTorch doesn't yet have 3.10 support on Windows (https://pytorch.org/get-started/locally/#windows-python) - os: macos-latest - python-version: '3.10' + python-version: '3.10.6' steps: - name: Checkout code @@ -51,7 +51,7 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install --upgrade --upgrade-strategy eager -r requirements/dev.txt python -m pip install --upgrade --upgrade-strategy eager -e . - if [ "$RUNNER_OS" != "Windows" ] && ${{ matrix.python-version != '3.10' }}; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. + if [ "$RUNNER_OS" != "Windows" ] && ${{ matrix.python-version != '3.10.6' }}; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. python -m pip install --upgrade --upgrade-strategy eager -e .[prophet] fi if [ "$RUNNER_OS" == "Linux" ]; then # Currently, we only support KeOps on Linux. From 3add1ebe304608fe0dc96bf4018076ebcc688ad1 Mon Sep 17 00:00:00 2001 From: Janis Klaise Date: Thu, 15 Sep 2022 14:39:50 +0100 Subject: [PATCH 13/35] Run test_changed_notebooks on push/PR to any branch and manual dispatch (#621) --- .github/workflows/test_changed_notebooks.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_changed_notebooks.yml b/.github/workflows/test_changed_notebooks.yml index d32e1dea0..a591c1dc6 100644 --- a/.github/workflows/test_changed_notebooks.yml +++ b/.github/workflows/test_changed_notebooks.yml @@ -7,17 +7,17 @@ defaults: shell: bash # To override PowerShell on Windows on: + # Trigger the workflow on push or PR to any branch push: - branches: - - master paths: - 'doc/source/examples/**/*.ipynb' pull_request: - branches: - - master paths: - 'doc/source/examples/**/*.ipynb' + # don't trigger for draft PRs types: [ opened, synchronize, reopened, ready_for_review ] + # Trigger the workflow on manual dispatch + workflow_dispatch: jobs: test_changed_notebooks: From f6b61d9ba81aec20395c47b7003fd79b5258fe36 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Fri, 16 Sep 2022 13:53:13 +0100 Subject: [PATCH 14/35] Update licences (#622) --- licenses/license.txt | 82 +++++++++++++++++++-------- licenses/license_info.csv | 47 +++++++-------- licenses/license_info.no_versions.csv | 1 + 3 files changed, 83 insertions(+), 47 deletions(-) diff --git a/licenses/license.txt b/licenses/license.txt index 2ae2c4c91..8346d2ba7 100644 --- a/licenses/license.txt +++ b/licenses/license.txt @@ -639,7 +639,7 @@ the file ChangeLog history information documenting your changes. Please read the FAQ for more information on the distribution of modified source versions. PyWavelets -1.3.0 +1.4.0 MIT License Copyright (c) 2006-2012 Filip Wasilewski Copyright (c) 2012-2020 The PyWavelets Developers @@ -689,7 +689,7 @@ SOFTWARE. alibi-detect -0.10.1.dev0 +0.10.4.dev0 Apache Software License Apache License Version 2.0, January 2004 @@ -921,7 +921,7 @@ SOFTWARE. certifi -2022.6.15 +2022.9.14 Mozilla Public License 2.0 (MPL 2.0) This package contains a modified version of ca-bundle.crt: @@ -931,7 +931,7 @@ Certificate data from Mozilla as of: Thu Nov 3 19:04:19 2011# This is a bundle of X.509 certificates of public Certificate Authorities (CA). These were automatically extracted from Mozilla's root certificates file (certdata.txt). This file can be found in the mozilla source tree: -http://mxr.mozilla.org/mozilla/source/security/nss/lib/ckfw/builtins/certdata.txt?raw=1# +https://hg.mozilla.org/mozilla-central/file/tip/security/nss/lib/ckfw/builtins/certdata.txt It contains the certificates in PEM format and therefore can be directly used with curl / libcurl / php_curl, or with an Apache+mod_ssl webserver for SSL client authentication. @@ -947,7 +947,7 @@ one at http://mozilla.org/MPL/2.0/. charset-normalizer -2.1.0 +2.1.1 MIT License MIT License @@ -971,6 +971,40 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +contourpy +1.0.5 +BSD License +BSD 3-Clause License + +Copyright (c) 2021-2022, ContourPy Developers. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + cycler 0.11.0 BSD License @@ -1048,7 +1082,7 @@ ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. filelock -3.7.1 +3.8.0 Public Domain This is free and unencumbered software released into the public domain. @@ -1077,7 +1111,7 @@ For more information, please refer to fonttools -4.34.4 +4.37.2 MIT License MIT License @@ -1103,7 +1137,7 @@ SOFTWARE. fsspec -2022.7.1 +2022.8.2 BSD License BSD 3-Clause License @@ -1137,7 +1171,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. huggingface-hub -0.8.1 +0.9.1 Apache Software License Apache License Version 2.0, January 2004 @@ -1343,7 +1377,7 @@ Apache Software License idna -3.3 +3.4 BSD License BSD 3-Clause License @@ -1404,7 +1438,7 @@ THE SOFTWARE. imageio -2.21.0 +2.21.3 BSD License Copyright (c) 2014-2022, imageio developers All rights reserved. @@ -1640,7 +1674,7 @@ Apache Software License joblib -1.1.0 +1.2.0 BSD License BSD 3-Clause License @@ -1806,7 +1840,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. matplotlib -3.5.2 +3.6.0 Python Software Foundation License License agreement for matplotlib versions 1.3.0 and later ========================================================= @@ -1909,7 +1943,7 @@ Licensee agrees to be bound by the terms and conditions of this License Agreement. networkx -2.8.5 +2.8.6 BSD License NetworkX is distributed with the 3-clause BSD license. @@ -5345,7 +5379,7 @@ under the terms of *both* these licenses. pandas -1.4.3 +1.4.4 BSD License BSD 3-Clause License @@ -5381,7 +5415,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. partd -1.2.0 +1.3.0 BSD Copyright (c) 2015, Continuum Analytics, Inc. and contributors All rights reserved. @@ -5414,7 +5448,7 @@ THE POSSIBILITY OF SUCH DAMAGE. pydantic -1.9.1 +1.10.2 MIT License The MIT License (MIT) @@ -5521,7 +5555,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. The above BSD License Applies to all code, even that also covered by Apache 2.0. pytz -2022.1 +2022.2.1 MIT License Copyright (c) 2003-2019 Stuart Bishop @@ -5545,7 +5579,7 @@ DEALINGS IN THE SOFTWARE. regex -2022.7.25 +2022.9.13 Apache Software License This work was derived from the 're' module of CPython 2.6 and CPython 3.1, copyright (c) 1998-2001 by Secret Labs AB and licensed under CNRI's Python 1.6 @@ -6058,7 +6092,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. scipy -1.9.0 +1.9.1 BSD License Copyright (c) 2001-2002 Enthought, Inc. 2003-2022, SciPy Developers. All rights reserved. @@ -7024,7 +7058,7 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. tifffile -2022.8.3 +2022.8.12 BSD License BSD 3-Clause License @@ -7128,7 +7162,7 @@ DAMAGE. tqdm -4.64.0 +4.64.1 MIT License; Mozilla Public License 2.0 (MPL 2.0) `tqdm` is a product of collaborative work. Unless otherwise stated, all authors (see commit logs) retain copyright @@ -7182,7 +7216,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. transformers -4.21.1 +4.22.0 Apache Software License Copyright 2018- The Hugging Face team. All rights reserved. @@ -7649,7 +7683,7 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. urllib3 -1.26.11 +1.26.12 MIT License MIT License diff --git a/licenses/license_info.csv b/licenses/license_info.csv index 14ac26631..3d1bb20cf 100644 --- a/licenses/license_info.csv +++ b/licenses/license_info.csv @@ -1,51 +1,52 @@ "Name","Version","License" "Pillow","9.2.0","Historical Permission Notice and Disclaimer (HPND)" -"PyWavelets","1.3.0","MIT License" +"PyWavelets","1.4.0","MIT License" "PyYAML","6.0","MIT License" -"alibi-detect","0.10.1.dev0","Apache Software License" +"alibi-detect","0.10.4.dev0","Apache Software License" "catalogue","2.0.8","MIT License" -"certifi","2022.6.15","Mozilla Public License 2.0 (MPL 2.0)" -"charset-normalizer","2.1.0","MIT License" +"certifi","2022.9.14","Mozilla Public License 2.0 (MPL 2.0)" +"charset-normalizer","2.1.1","MIT License" +"contourpy","1.0.5","BSD License" "cycler","0.11.0","BSD License" "dataclasses","0.6","Apache Software License" "dill","0.3.5.1","BSD License" -"filelock","3.7.1","Public Domain" -"fonttools","4.34.4","MIT License" -"fsspec","2022.7.1","BSD License" -"huggingface-hub","0.8.1","Apache Software License" -"idna","3.3","BSD License" +"filelock","3.8.0","Public Domain" +"fonttools","4.37.2","MIT License" +"fsspec","2022.8.2","BSD License" +"huggingface-hub","0.9.1","Apache Software License" +"idna","3.4","BSD License" "idna-ssl","1.1.0","MIT License" -"imageio","2.21.0","BSD License" +"imageio","2.21.3","BSD License" "importlib-metadata","4.12.0","Apache Software License" -"joblib","1.1.0","BSD License" +"joblib","1.2.0","BSD License" "kiwisolver","1.4.4","BSD License" "llvmlite","0.38.1","BSD" "locket","1.0.0","BSD License" -"matplotlib","3.5.2","Python Software Foundation License" -"networkx","2.8.5","BSD License" +"matplotlib","3.6.0","Python Software Foundation License" +"networkx","2.8.6","BSD License" "numba","0.55.2","BSD License" "numpy","1.22.4","BSD License" "opencv-python","4.6.0.66","MIT License" "packaging","21.3","Apache Software License; BSD License" -"pandas","1.4.3","BSD License" -"partd","1.2.0","BSD" -"pydantic","1.9.1","MIT License" +"pandas","1.4.4","BSD License" +"partd","1.3.0","BSD" +"pydantic","1.10.2","MIT License" "pyparsing","3.0.9","MIT License" "python-dateutil","2.8.2","Apache Software License; BSD License" -"pytz","2022.1","MIT License" -"regex","2022.7.25","Apache Software License" +"pytz","2022.2.1","MIT License" +"regex","2022.9.13","Apache Software License" "requests","2.28.1","Apache Software License" "scikit-image","0.19.3","BSD License" "scikit-learn","1.1.2","BSD License" -"scipy","1.9.0","BSD License" +"scipy","1.9.1","BSD License" "six","1.16.0","MIT License" "threadpoolctl","3.1.0","BSD License" -"tifffile","2022.8.3","BSD License" +"tifffile","2022.8.12","BSD License" "tokenizers","0.12.1","Apache Software License" "toml","0.10.2","MIT License" "toolz","0.12.0","BSD License" -"tqdm","4.64.0","MIT License; Mozilla Public License 2.0 (MPL 2.0)" -"transformers","4.21.1","Apache Software License" +"tqdm","4.64.1","MIT License; Mozilla Public License 2.0 (MPL 2.0)" +"transformers","4.22.0","Apache Software License" "typing-extensions","4.3.0","Python Software Foundation License" -"urllib3","1.26.11","MIT License" +"urllib3","1.26.12","MIT License" "zipp","3.8.1","MIT License" \ No newline at end of file diff --git a/licenses/license_info.no_versions.csv b/licenses/license_info.no_versions.csv index 0c164f9f8..0d61fe837 100644 --- a/licenses/license_info.no_versions.csv +++ b/licenses/license_info.no_versions.csv @@ -6,6 +6,7 @@ "catalogue","MIT License" "certifi","Mozilla Public License 2.0 (MPL 2.0)" "charset-normalizer","MIT License" +"contourpy","BSD License" "cycler","BSD License" "dataclasses","Apache Software License" "dill","BSD License" From 53fa1c13a3025a5f0569845d84da3d98a1523a71 Mon Sep 17 00:00:00 2001 From: RobertSamoilescu Date: Wed, 21 Sep 2022 10:06:08 +0100 Subject: [PATCH 15/35] Revert "Tabular classifier drift example - dummy trap (#557)" (#624) This reverts commit 4e2419877f89cb7e72721644a947f8b390f123fb. --- doc/source/examples/cd_clf_adult.ipynb | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/doc/source/examples/cd_clf_adult.ipynb b/doc/source/examples/cd_clf_adult.ipynb index 265b7c84f..7aedc4976 100644 --- a/doc/source/examples/cd_clf_adult.ipynb +++ b/doc/source/examples/cd_clf_adult.ipynb @@ -206,10 +206,10 @@ "# define numerical standard scaler.\n", "num_transf = StandardScaler()\n", "\n", - "# define categorical one-hot encoder with first columns dropped.\n", + "# define categorical one-hot encoder.\n", "cat_transf = OneHotEncoder(\n", " categories=[range(len(x)) for x in adult.category_map.values()],\n", - " drop=\"first\",\n", + " handle_unknown=\"ignore\"\n", ")\n", "\n", "# Define column transformer\n", @@ -286,7 +286,7 @@ "text": [ "H0\n", "Drift? No!\n", - "p-value: 0.665\n", + "p-value: 0.681\n", "\n", "H1\n", "Drift? Yes!\n", @@ -357,6 +357,7 @@ "# define model\n", "model = GradientBoostingClassifier()\n", "\n", + "\n", "# define drift detector\n", "detector = ClassifierDrift(\n", " x_ref=x_ref,\n", @@ -401,7 +402,7 @@ "text": [ "H0\n", "Drift? No!\n", - "p-value: 0.359\n", + "p-value: 0.457\n", "\n", "H1\n", "Drift? Yes!\n", @@ -483,14 +484,14 @@ "text": [ "H0\n", "Drift? No!\n", - "p-value: 0.905\n", + "p-value: 0.670\n", "\n", "H1\n", "Drift? Yes!\n", "p-value: 0.000\n", "\n", - "CPU times: user 8.68 s, sys: 42.2 ms, total: 8.72 s\n", - "Wall time: 8.72 s\n" + "CPU times: user 5.13 s, sys: 4.92 ms, total: 5.14 s\n", + "Wall time: 5.13 s\n" ] } ], @@ -531,14 +532,14 @@ "text": [ "H0\n", "Drift? No!\n", - "p-value: 0.952\n", + "p-value: 0.905\n", "\n", "H1\n", "Drift? Yes!\n", "p-value: 0.000\n", "\n", - "CPU times: user 2.02 s, sys: 12.5 ms, total: 2.03 s\n", - "Wall time: 2.03 s\n" + "CPU times: user 1.39 s, sys: 18.3 ms, total: 1.41 s\n", + "Wall time: 1.41 s\n" ] } ], From 6274a96fbe56ff2ed4e44199c4634917212d3505 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Sep 2022 16:00:15 +0100 Subject: [PATCH 16/35] Bump sphinx-design from 0.2.0 to 0.3.0 (#599) Bumps [sphinx-design](https://github.com/executablebooks/sphinx-design) from 0.2.0 to 0.3.0. - [Release notes](https://github.com/executablebooks/sphinx-design/releases) - [Changelog](https://github.com/executablebooks/sphinx-design/blob/main/CHANGELOG.md) - [Commits](https://github.com/executablebooks/sphinx-design/compare/v0.2.0...v0.3.0) --- updated-dependencies: - dependency-name: sphinx-design dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/docs.txt b/requirements/docs.txt index 435aadc29..aae15bacb 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -7,7 +7,7 @@ sphinxcontrib-apidoc>=0.3.0, <0.4.0 sphinxcontrib-bibtex>=2.1.0, <3.0.0 myst-parser>=0.14, <0.19 nbsphinx>=0.8.5, <0.9.0 -sphinx_design==0.2.0 # Pinning for now as sphinx_design is v.new and still in flux. +sphinx_design==0.3.0 # Pinning for now as sphinx_design is v.new and still in flux. ipykernel>=5.1.0, <7.0.0 # required for executing notebooks via nbsphinx ipython>=7.2.0, <9.0.0 # required for executing notebooks nbsphinx # pandoc From a08e2e61dd14d32d933a3f1170bf16164dd3751b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 22 Sep 2022 14:43:42 +0100 Subject: [PATCH 17/35] Update nbconvert requirement from <7.0.0,>=6.0.7 to >=6.0.7,<8.0.0 (#598) Updates the requirements on [nbconvert](https://github.com/jupyter/nbconvert) to permit the latest version. - [Release notes](https://github.com/jupyter/nbconvert/releases) - [Commits](https://github.com/jupyter/nbconvert/compare/6.0.7...7.0.0) --- updated-dependencies: - dependency-name: nbconvert dependency-type: direct:development ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 114f0b9f4..812dc6fb5 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -14,7 +14,7 @@ pytest-custom_exit_code>=0.3.0 # for notebook tests pytest-timeout>=1.4.2, <3.0.0 # for notebook tests jupytext>=1.12.0, <2.0.0 # for notebook tests ipykernel>=5.1.0, <7.0.0 # for notebook tests -nbconvert>=6.0.7, <7.0.0 # for notebook tests +nbconvert>=6.0.7, <8.0.0 # for notebook tests ipywidgets>=7.6.5, <8.0.0 # for notebook tests alibi-testing @ git+https://github.com/SeldonIO/alibi-testing@master#egg=alibi-testing # pre-trained models for testing # other From 341af20c5e7e33ab46a206e1d2313c0465380085 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 22 Sep 2022 14:46:16 +0100 Subject: [PATCH 18/35] Update ipywidgets requirement from <8.0.0,>=7.6.5 to >=7.6.5,<9.0.0 (#592) Updates the requirements on [ipywidgets](https://github.com/jupyter-widgets/ipywidgets) to permit the latest version. - [Release notes](https://github.com/jupyter-widgets/ipywidgets/releases) - [Commits](https://github.com/jupyter-widgets/ipywidgets/compare/7.6.5...8.0.1) --- updated-dependencies: - dependency-name: ipywidgets dependency-type: direct:development ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Janis Klaise --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 812dc6fb5..587b4363c 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -15,7 +15,7 @@ pytest-timeout>=1.4.2, <3.0.0 # for notebook tests jupytext>=1.12.0, <2.0.0 # for notebook tests ipykernel>=5.1.0, <7.0.0 # for notebook tests nbconvert>=6.0.7, <8.0.0 # for notebook tests -ipywidgets>=7.6.5, <8.0.0 # for notebook tests +ipywidgets>=7.6.5, <9.0.0 # for notebook tests alibi-testing @ git+https://github.com/SeldonIO/alibi-testing@master#egg=alibi-testing # pre-trained models for testing # other pre-commit>=1.20.0, <3.0.0 From 2683721fbbdae8b9447d15d35f416281290863ed Mon Sep 17 00:00:00 2001 From: Janis Klaise Date: Tue, 27 Sep 2022 11:03:01 +0100 Subject: [PATCH 19/35] Reraise original error when handling missing dependencies (#630) --- alibi_detect/utils/missing_optional_dependency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alibi_detect/utils/missing_optional_dependency.py b/alibi_detect/utils/missing_optional_dependency.py index d31d35fa7..10d6755fd 100644 --- a/alibi_detect/utils/missing_optional_dependency.py +++ b/alibi_detect/utils/missing_optional_dependency.py @@ -107,7 +107,7 @@ def import_optional(module_name: str, names: Optional[List[str]] = None) -> Any: return module except (ImportError, ModuleNotFoundError) as err: if err.name is None: - raise TypeError() + raise err dep_name, *_ = err.name.split('.') if str(dep_name) not in ERROR_TYPES: raise err From c7be43091b8a6662825be2eadb607fed2c3b83f7 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Tue, 27 Sep 2022 14:36:20 +0100 Subject: [PATCH 20/35] Limit sphinx-autodoc-typehints upper bound (#633) --- requirements/docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/docs.txt b/requirements/docs.txt index aae15bacb..5895093aa 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,7 +1,7 @@ # dependencies for building docs, separate from dev.txt as this is also used for builds on readthedocs.org # core dependencies sphinx>=4.2.0, <5.1.0 -sphinx-autodoc-typehints>=1.12.0, <2.0.0 +sphinx-autodoc-typehints>=1.12.0, <1.19.3 # limited due to https://github.com/tox-dev/sphinx-autodoc-typehints/issues/259 and 260 sphinx-rtd-theme>=1.0.0, <2.0.0 sphinxcontrib-apidoc>=0.3.0, <0.4.0 sphinxcontrib-bibtex>=2.1.0, <3.0.0 From 75171129c9ec1da4d9055f8b2c9cde0122542c10 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Tue, 27 Sep 2022 14:56:41 +0100 Subject: [PATCH 21/35] Fix numbered list in docstring (#636) --- alibi_detect/utils/missing_optional_dependency.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/alibi_detect/utils/missing_optional_dependency.py b/alibi_detect/utils/missing_optional_dependency.py index 10d6755fd..304475692 100644 --- a/alibi_detect/utils/missing_optional_dependency.py +++ b/alibi_detect/utils/missing_optional_dependency.py @@ -20,10 +20,11 @@ """Mapping used to ensure correct pip install message is generated if a missing optional dependency is detected. This dict is used to control two behaviours: - 1. When we import objects from missing dependencies we check that any `ModuleNotFoundError` or `ImportError` + +1. When we import objects from missing dependencies we check that any `ModuleNotFoundError` or `ImportError` corresponds to a missing optional dependency by checking the name of the missing dependency is in `ERROR_TYPES`. We then map this name to the corresponding optional dependency bucket that will resolve the issue. - 2. Some optional dependencies have multiple names such as `torch` and `pytorch`, instead of enforcing a single +2. Some optional dependencies have multiple names such as `torch` and `pytorch`, instead of enforcing a single naming convention across the whole code base we instead use `ERROR_TYPES` to capture both cases. This is done right before the pip install message is issued as this is the most robust place to capture these differences. """ From df7edca2f565b241062c5fdf4811ef50821e3ce6 Mon Sep 17 00:00:00 2001 From: Janis Klaise Date: Tue, 27 Sep 2022 16:33:29 +0100 Subject: [PATCH 22/35] Dev/upgrade prophet 1.1 (#627) --- .github/workflows/ci.yml | 17 +++++++---------- .github/workflows/test_all_notebooks.yml | 5 +---- .github/workflows/test_changed_notebooks.yml | 5 +---- CHANGELOG.md | 5 +++-- alibi_detect/od/prophet.py | 2 +- alibi_detect/od/tests/test_prophet.py | 4 ++-- .../utils/missing_optional_dependency.py | 4 +--- setup.py | 8 ++------ 8 files changed, 18 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82ada6464..4f242eb2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ on: branches: - master - 'feature/*' - types: [opened, synchronize, reopened, ready_for_review] + types: [ opened, synchronize, reopened, ready_for_review ] # Trigger workflow once per week schedule: - cron: '0 0 * * *' @@ -29,13 +29,13 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest] + os: [ ubuntu-latest ] python-version: [ '3.7', '3.8', '3.9', '3.10.6' ] include: # Run macos and windows tests on only one python version - - os: windows-latest - python-version: '3.9' # PyTorch doesn't yet have 3.10 support on Windows (https://pytorch.org/get-started/locally/#windows-python) - - os: macos-latest - python-version: '3.10.6' + - os: windows-latest + python-version: '3.9' # PyTorch doesn't yet have 3.10 support on Windows (https://pytorch.org/get-started/locally/#windows-python) + - os: macos-latest + python-version: '3.10.6' steps: - name: Checkout code @@ -51,13 +51,10 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install --upgrade --upgrade-strategy eager -r requirements/dev.txt python -m pip install --upgrade --upgrade-strategy eager -e . - if [ "$RUNNER_OS" != "Windows" ] && ${{ matrix.python-version != '3.10.6' }}; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. - python -m pip install --upgrade --upgrade-strategy eager -e .[prophet] - fi if [ "$RUNNER_OS" == "Linux" ]; then # Currently, we only support KeOps on Linux. python -m pip install --upgrade --upgrade-strategy eager -e .[keops] fi - python -m pip install --upgrade --upgrade-strategy eager -e .[tensorflow,torch] + python -m pip install --upgrade --upgrade-strategy eager -e .[prophet,tensorflow,torch] python -m pip freeze - name: Lint with flake8 diff --git a/.github/workflows/test_all_notebooks.yml b/.github/workflows/test_all_notebooks.yml index 5526a51f7..9ab63f0e8 100644 --- a/.github/workflows/test_all_notebooks.yml +++ b/.github/workflows/test_all_notebooks.yml @@ -41,10 +41,7 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install --upgrade --upgrade-strategy eager -r requirements/dev.txt -r testing/requirements.txt python -m pip install --upgrade --upgrade-strategy eager -e . - if [ "$RUNNER_OS" != "Windows" ] && ${{ matrix.python-version != '3.10' }}; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. - python -m pip install --upgrade --upgrade-strategy eager -e .[prophet] - fi - python -m pip install --upgrade --upgrade-strategy eager -e .[torch,tensorflow] + python -m pip install --upgrade --upgrade-strategy eager -e .[prophet,torch,tensorflow] python -m pip freeze - name: Run notebooks diff --git a/.github/workflows/test_changed_notebooks.yml b/.github/workflows/test_changed_notebooks.yml index a591c1dc6..6637e68bb 100644 --- a/.github/workflows/test_changed_notebooks.yml +++ b/.github/workflows/test_changed_notebooks.yml @@ -56,10 +56,7 @@ jobs: python -m pip install --upgrade pip setuptools wheel python -m pip install --upgrade --upgrade-strategy eager -r requirements/dev.txt -r testing/requirements.txt python -m pip install --upgrade --upgrade-strategy eager -e . - if [ "$RUNNER_OS" != "Windows" ] && ${{ matrix.python-version != '3.10' }}; then # Skip Prophet tests on Windows as installation complex. Skip on Python 3.10 as not supported. - python -m pip install --upgrade --upgrade-strategy eager -e .[prophet] - fi - python -m pip install --upgrade --upgrade-strategy eager -e .[torch,tensorflow] + python -m pip install --upgrade --upgrade-strategy eager -e .[prophet,torch,tensorflow] python -m pip freeze - name: Run notebooks diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f4b7fef9..8ebcc88b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,18 +8,19 @@ See the [documentation](https://docs.seldon.io/projects/alibi-detect/en/latest/cd/methods/mmddrift.html) and [example notebook](https://docs.seldon.io/projects/alibi-detect/en/latest/examples/cd_mmd_keops.html) for more info ([#548](https://github.com/SeldonIO/alibi-detect/pull/548)). - If a `categories_per_feature` dictionary is not passed to `TabularDrift`, a warning is now raised to inform the user that all features are assumed to be numerical ([#606](https://github.com/SeldonIO/alibi-detect/pull/606)). +### Changed +- Minimum `prophet` version bumped to `1.1.0` (used by `OutlierProphet`). This upgrade removes the dependency on `pystan` as `cmdstanpy` is used instead. This version also comes with pre-built wheels for all major platforms and Python versions, making both installation and testing easier ([#627](https://github.com/SeldonIO/alibi-detect/pull/627)). + ### Development - UTF-8 decoding is enforced when `README.md` is opened by `setup.py`. This is to prevent pip install errors on systems with `PYTHONIOENCODING` set to use other encoders ([#605](https://github.com/SeldonIO/alibi-detect/pull/605)). - Skip specific save/load tests that require downloading remote artefacts if the relevant URI(s) is/are down ([#607](https://github.com/SeldonIO/alibi-detect/pull/607)). -## v0.10.3 ## [v0.10.3](https://github.com/SeldonIO/alibi-detect/tree/v0.10.3) (2022-08-17) [Full Changelog](https://github.com/SeldonIO/alibi-detect/compare/v0.10.2...v0.10.3) ### Fixed - Fix to allow `config.toml` files to be loaded when the [meta] field is not present ([#591](https://github.com/SeldonIO/alibi-detect/pull/591)). -## v0.10.2 ## [v0.10.2](https://github.com/SeldonIO/alibi-detect/tree/v0.10.2) (2022-08-16) [Full Changelog](https://github.com/SeldonIO/alibi-detect/compare/v0.10.1...v0.10.2) diff --git a/alibi_detect/od/prophet.py b/alibi_detect/od/prophet.py index 2c0d90865..787517791 100644 --- a/alibi_detect/od/prophet.py +++ b/alibi_detect/od/prophet.py @@ -1,4 +1,4 @@ -from fbprophet import Prophet +from prophet import Prophet import logging import pandas as pd from typing import Dict, List, Union diff --git a/alibi_detect/od/tests/test_prophet.py b/alibi_detect/od/tests/test_prophet.py index ef1c7169c..9ccd2c11d 100644 --- a/alibi_detect/od/tests/test_prophet.py +++ b/alibi_detect/od/tests/test_prophet.py @@ -33,10 +33,10 @@ def prophet_params(request): @pytest.mark.parametrize('prophet_params', list(range(n_tests)), indirect=True) def test_prophet(prophet_params): - fbprophet = pytest.importorskip('fbprophet', reason="Prophet tests skipped as Prophet not installed") + prophet = pytest.importorskip('prophet', reason="Prophet tests skipped as Prophet not installed") growth, return_instance_score, return_forecast = prophet_params od = OutlierProphet(growth=growth) - assert isinstance(od.model, fbprophet.forecaster.Prophet) + assert isinstance(od.model, prophet.forecaster.Prophet) assert od.meta == {'name': 'OutlierProphet', 'detector_type': 'outlier', 'data_type': 'time-series', 'online': False, 'version': __version__} if growth == 'logistic': diff --git a/alibi_detect/utils/missing_optional_dependency.py b/alibi_detect/utils/missing_optional_dependency.py index 304475692..832923a24 100644 --- a/alibi_detect/utils/missing_optional_dependency.py +++ b/alibi_detect/utils/missing_optional_dependency.py @@ -29,9 +29,7 @@ before the pip install message is issued as this is the most robust place to capture these differences. """ ERROR_TYPES = { - "fbprophet": 'prophet', - "holidays": 'prophet', - "pystan": 'prophet', + "prophet": 'prophet', "tensorflow_probability": 'tensorflow', "tensorflow": 'tensorflow', "torch": 'torch', diff --git a/setup.py b/setup.py index 4907088d8..080ca56b4 100644 --- a/setup.py +++ b/setup.py @@ -11,9 +11,7 @@ def readme(): extras_require = { "prophet": [ - "fbprophet>=0.5, <0.7", - "holidays==0.9.11", - "pystan<3.0" + "prophet>=1.1.0, <2.0.0", ], "torch": [ "torch>=1.7.0, <1.13.0" @@ -28,9 +26,7 @@ def readme(): "torch>=1.7.0, <1.13.0" ], "all": [ - "fbprophet>=0.5, <0.7", - "holidays==0.9.11", - "pystan<3.0", + "prophet>=1.1.0, <2.0.0", "tensorflow_probability>=0.8.0, <0.18.0", "tensorflow>=2.2.0, !=2.6.0, !=2.6.1, <2.10.0", # https://github.com/SeldonIO/alibi-detect/issues/375 and 387 "pykeops>=2.0.0, <2.2.0", From 4af80a12a96cc203a79839d42df48390033c7ad9 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Thu, 29 Sep 2022 10:13:19 +0100 Subject: [PATCH 23/35] Minor save/load refactoring (#635) --- alibi_detect/base.py | 25 ++++++---- .../{tensorflow => _tensorflow}/__init__.py | 18 +++++-- .../conversions.py} | 0 .../_loading.py => _tensorflow/loading.py} | 0 .../_saving.py => _tensorflow/saving.py} | 6 +-- alibi_detect/saving/loading.py | 4 +- alibi_detect/saving/saving.py | 2 +- alibi_detect/saving/tests/test_saving.py | 50 ++++++++++--------- alibi_detect/tests/test_dep_management.py | 4 +- doc/source/overview/saving.md | 39 ++++++++++++++- 10 files changed, 104 insertions(+), 44 deletions(-) rename alibi_detect/saving/{tensorflow => _tensorflow}/__init__.py (60%) rename alibi_detect/saving/{tensorflow/_conversions.py => _tensorflow/conversions.py} (100%) rename alibi_detect/saving/{tensorflow/_loading.py => _tensorflow/loading.py} (100%) rename alibi_detect/saving/{tensorflow/_saving.py => _tensorflow/saving.py} (100%) diff --git a/alibi_detect/base.py b/alibi_detect/base.py index db1ef8067..847220f03 100644 --- a/alibi_detect/base.py +++ b/alibi_detect/base.py @@ -119,16 +119,14 @@ def get_config(self) -> dict: # TODO - move to BaseDetector once config save/lo if self.config is not None: # Get config (stored in top-level self) cfg = self.config - # Get low-level nested detector (if needed) - detector = self._detector if hasattr(self, '_detector') else self # type: ignore[attr-defined] - detector = detector._detector if hasattr(detector, '_detector') else detector # type: ignore[attr-defined] # Add large artefacts back to config for key in LARGE_ARTEFACTS: - if key in cfg: # self.config is validated, therefore if a key is not in cfg, it isn't valid to insert - cfg[key] = getattr(detector, key) + if key in cfg and hasattr(self._nested_detector, key): + cfg[key] = getattr(self._nested_detector, key) # Set x_ref_preprocessed flag - preprocess_at_init = getattr(detector, 'preprocess_at_init', True) # If no preprocess_at_init, always true! - cfg['x_ref_preprocessed'] = preprocess_at_init and detector.preprocess_fn is not None + # If no preprocess_at_init, always true! + preprocess_at_init = getattr(self._nested_detector, 'preprocess_at_init', True) + cfg['x_ref_preprocessed'] = preprocess_at_init and self._nested_detector.preprocess_fn is not None return cfg else: raise NotImplementedError('Getting a config (or saving via a config file) is not yet implemented for this' @@ -185,17 +183,26 @@ def _set_config(self, inputs): # TODO - move to BaseDetector once config save/l # Overwrite any large artefacts with None to save memory. They'll be added back by get_config() for key in LARGE_ARTEFACTS: - if key in inputs: + if key in inputs and hasattr(self._nested_detector, key): inputs[key] = None self.config.update(inputs) + @property + def _nested_detector(self): + """ + The low-level nested detector. + """ + detector = self._detector if hasattr(self, '_detector') else self # type: ignore[attr-defined] + detector = detector._detector if hasattr(detector, '_detector') else detector # type: ignore[attr-defined] + return detector + @runtime_checkable class Detector(Protocol): """Type Protocol for all detectors. - Used for typing legacy save and load functionality in `alibi_detect.saving.tensorflow._saving.py`. + Used for typing legacy save and load functionality in `alibi_detect.saving._tensorflow.saving.py`. Note: This exists to distinguish between detectors with and without support for config saving and loading. Once all diff --git a/alibi_detect/saving/tensorflow/__init__.py b/alibi_detect/saving/_tensorflow/__init__.py similarity index 60% rename from alibi_detect/saving/tensorflow/__init__.py rename to alibi_detect/saving/_tensorflow/__init__.py index 3d4f1f84c..e2f8220a5 100644 --- a/alibi_detect/saving/tensorflow/__init__.py +++ b/alibi_detect/saving/_tensorflow/__init__.py @@ -2,7 +2,7 @@ load_detector_legacy, load_kernel_config_tf, load_embedding_tf, load_model_tf, load_optimizer_tf, \ prep_model_and_emb_tf = import_optional( - 'alibi_detect.saving.tensorflow._loading', + 'alibi_detect.saving._tensorflow.loading', names=['load_detector_legacy', 'load_kernel_config', 'load_embedding', @@ -11,11 +11,23 @@ 'prep_model_and_emb']) save_detector_legacy, save_model_config_tf = import_optional( - 'alibi_detect.saving.tensorflow._saving', + 'alibi_detect.saving._tensorflow.saving', names=['save_detector_legacy', 'save_model_config'] ) get_tf_dtype = import_optional( - 'alibi_detect.saving.tensorflow._conversions', + 'alibi_detect.saving._tensorflow.conversions', names=['get_tf_dtype'] ) + +__all__ = [ + "load_detector_legacy", + "load_kernel_config_tf", + "load_embedding_tf", + "load_model_tf", + "load_optimizer_tf", + "prep_model_and_emb_tf", + "save_detector_legacy", + "save_model_config_tf", + "get_tf_dtype" +] diff --git a/alibi_detect/saving/tensorflow/_conversions.py b/alibi_detect/saving/_tensorflow/conversions.py similarity index 100% rename from alibi_detect/saving/tensorflow/_conversions.py rename to alibi_detect/saving/_tensorflow/conversions.py diff --git a/alibi_detect/saving/tensorflow/_loading.py b/alibi_detect/saving/_tensorflow/loading.py similarity index 100% rename from alibi_detect/saving/tensorflow/_loading.py rename to alibi_detect/saving/_tensorflow/loading.py diff --git a/alibi_detect/saving/tensorflow/_saving.py b/alibi_detect/saving/_tensorflow/saving.py similarity index 100% rename from alibi_detect/saving/tensorflow/_saving.py rename to alibi_detect/saving/_tensorflow/saving.py index 8eeb6df66..1b4ef7126 100644 --- a/alibi_detect/saving/tensorflow/_saving.py +++ b/alibi_detect/saving/_tensorflow/saving.py @@ -150,6 +150,9 @@ def save_embedding_config(embed: TransformerEmbedding, return cfg_embed +####################################################################################################### +# TODO: Everything below here is legacy saving code, and will be removed in the future +####################################################################################################### def save_embedding_legacy(embed: TransformerEmbedding, embed_args: dict, filepath: Path) -> None: @@ -177,9 +180,6 @@ def save_embedding_legacy(embed: TransformerEmbedding, dill.dump(embed_args, f) -####################################################################################################### -# TODO: Everything below here is legacy saving code, and will be removed in the future -####################################################################################################### def save_detector_legacy(detector, filepath): detector_name = detector.meta['name'] diff --git a/alibi_detect/saving/loading.py b/alibi_detect/saving/loading.py index 2d53dd3ac..88c67e0a6 100644 --- a/alibi_detect/saving/loading.py +++ b/alibi_detect/saving/loading.py @@ -12,7 +12,7 @@ from transformers import AutoTokenizer from alibi_detect.saving.registry import registry -from alibi_detect.saving.tensorflow import load_detector_legacy, load_embedding_tf, load_kernel_config_tf, \ +from alibi_detect.saving._tensorflow import load_detector_legacy, load_embedding_tf, load_kernel_config_tf, \ load_model_tf, load_optimizer_tf, prep_model_and_emb_tf, get_tf_dtype from alibi_detect.saving.validate import validate_config from alibi_detect.base import Detector, ConfigurableDetector @@ -129,7 +129,7 @@ def _load_detector_config(filepath: Union[str, os.PathLike]) -> ConfigurableDete # Backend backend = cfg.pop('backend') # popping so that cfg left as kwargs + `name` when passed to _init_detector if backend.lower() != 'tensorflow': - raise NotImplementedError('Loading detectors with PyTorch or sklearn backend is not yet supported.') + raise NotImplementedError('Loading detectors with PyTorch, sklearn or keops backend is not yet supported.') # Init detector from config logger.info('Instantiating detector.') diff --git a/alibi_detect/saving/saving.py b/alibi_detect/saving/saving.py index b1ae31983..80c9430dd 100644 --- a/alibi_detect/saving/saving.py +++ b/alibi_detect/saving/saving.py @@ -14,7 +14,7 @@ from alibi_detect.saving.loading import _replace, validate_config from alibi_detect.saving.registry import registry from alibi_detect.saving.schemas import SupportedModels -from alibi_detect.saving.tensorflow import save_detector_legacy, save_model_config_tf +from alibi_detect.saving._tensorflow import save_detector_legacy, save_model_config_tf from alibi_detect.base import Detector, ConfigurableDetector # do not extend pickle dispatch table so as not to change pickle behaviour diff --git a/alibi_detect/saving/tests/test_saving.py b/alibi_detect/saving/tests/test_saving.py index badf7c1c3..3d08cfd96 100644 --- a/alibi_detect/saving/tests/test_saving.py +++ b/alibi_detect/saving/tests/test_saving.py @@ -176,7 +176,7 @@ def deep_kernel(request, backend, encoder_model): @fixture -def classifier(backend, current_cases): +def classifier_model(backend, current_cases): """ Classification model with given input dimension and backend. """ @@ -204,7 +204,7 @@ def nlp_embedding_and_tokenizer(model_name, max_len, uae, backend): # Load tokenizer try: - tokenizer = AutoTokenizer.from_pretrained(model_name + 'TODO') + tokenizer = AutoTokenizer.from_pretrained(model_name) except (OSError, HTTPError): pytest.skip(f"Problem downloading {model_name} from huggingface.co") X = 'A dummy string' # this will be padded to max_len @@ -267,15 +267,18 @@ def preprocess_nlp(embedding, tokenizer, max_len, backend): @fixture -def preprocess_hiddenoutput(classifier, backend): +def preprocess_hiddenoutput(classifier_model, current_cases, backend): """ Preprocess function to extract the softmax layer of a classifier (with the HiddenOutput utility function). """ + _, _, data_params = current_cases["data"] + _, input_dim = data_params['data_shape'] + if backend == 'tensorflow': - model = HiddenOutput_tf(classifier, layer=-1) + model = HiddenOutput_tf(classifier_model, layer=-1, input_shape=(None, input_dim)) preprocess_fn = partial(preprocess_drift_tf, model=model) else: - model = HiddenOutput_pt(classifier, layer=-1) + model = HiddenOutput_pt(classifier_model, layer=-1) preprocess_fn = partial(preprocess_drift_pt, model=model) return preprocess_fn @@ -310,7 +313,7 @@ def test_load_simple_config(cfg, tmp_path): @parametrize_with_cases("data", cases=ContinuousData, prefix='data_') def test_save_ksdrift(data, preprocess_fn, tmp_path): """ - Test KSDrift on continuous datasets, with UAE and classifier softmax output as preprocess_fn's. Only this + Test KSDrift on continuous datasets, with UAE and classifier_model softmax output as preprocess_fn's. Only this detector is tested with preprocessing strategies, as other detectors should see the same preprocess_fn output. Detector is saved and then loaded, with assertions checking that the reinstantiated detector is equivalent. @@ -337,9 +340,9 @@ def test_save_ksdrift(data, preprocess_fn, tmp_path): @parametrize('preprocess_fn', [preprocess_nlp]) @parametrize_with_cases("data", cases=TextData.movie_sentiment_data, prefix='data_') -def test_save_ksdrift_nlp(data, preprocess_fn, max_len, enc_dim, tmp_path): +def test_save_ksdrift_nlp(data, preprocess_fn, enc_dim, tmp_path): """ - Test KSDrift on continuous datasets, with UAE and classifier softmax output as preprocess_fn's. Only this + Test KSDrift on continuous datasets, with UAE and classifier_model softmax output as preprocess_fn's. Only this detector is tested with embedding and embedding+uae, as other detectors should see the same preprocessed data. Detector is saved and then loaded, with assertions checking that the reinstantiated detector is equivalent. @@ -350,7 +353,7 @@ def test_save_ksdrift_nlp(data, preprocess_fn, max_len, enc_dim, tmp_path): p_val=P_VAL, preprocess_fn=preprocess_fn, preprocess_at_init=True, - input_shape=(max_len,), + input_shape=(768,), # hardcoded to bert-base-cased for now ) save_detector(cd, tmp_path, legacy=False) cd_load = load_detector(tmp_path) @@ -572,13 +575,13 @@ def test_save_tabulardrift(data, tmp_path): @parametrize_with_cases("data", cases=ContinuousData, prefix='data_') -def test_save_classifierdrift(data, classifier, backend, tmp_path, seed): +def test_save_classifierdrift(data, classifier_model, backend, tmp_path, seed): """ Test ClassifierDrift on continuous datasets.""" # Init detector and predict X_ref, X_h0 = data with fixed_seed(seed): cd = ClassifierDrift(X_ref, - model=classifier, + model=classifier_model, p_val=P_VAL, n_folds=5, backend=backend, @@ -606,7 +609,7 @@ def test_save_classifierdrift(data, classifier, backend, tmp_path, seed): @parametrize_with_cases("data", cases=ContinuousData, prefix='data_') -def test_save_spotthediff(data, classifier, backend, tmp_path, seed): +def test_save_spotthediff(data, classifier_model, backend, tmp_path, seed): """ Test SpotTheDiffDrift on continuous datasets. @@ -731,13 +734,13 @@ def test_save_contextmmddrift(data, kernel, backend, tmp_path, seed): @parametrize_with_cases("data", cases=ContinuousData, prefix='data_') -def test_save_classifieruncertaintydrift(data, classifier, backend, tmp_path, seed): +def test_save_classifieruncertaintydrift(data, classifier_model, backend, tmp_path, seed): """ Test ClassifierDrift on continuous datasets.""" # Init detector and predict X_ref, X_h0 = data with fixed_seed(seed): cd = ClassifierUncertaintyDrift(X_ref, - model=classifier, + model=classifier_model, p_val=P_VAL, backend=backend, preds_type='probs', @@ -1071,19 +1074,20 @@ def test_save_deepkernel(data, deep_kernel, backend, tmp_path): """ # Get data dim X, _ = data - input_dim = X.shape[1] + input_shape = (X.shape[1],) # Save kernel to config filepath = tmp_path filename = 'mykernel' cfg_kernel = _save_kernel_config(deep_kernel, filepath, filename) - cfg_kernel['proj'], _ = _save_model_config(cfg_kernel['proj'], base_path=filepath, input_shape=input_dim, + cfg_kernel['proj'], _ = _save_model_config(cfg_kernel['proj'], base_path=filepath, input_shape=input_shape, backend=backend) cfg_kernel = _path2str(cfg_kernel) - cfg_kernel['proj'] = ModelConfig(**cfg_kernel['proj']).dict() # Pass thru ModelConfig to set `custom_objects` etc + cfg_kernel['proj'] = ModelConfig(**cfg_kernel['proj']).dict() # Pass thru ModelConfig to set `layers` etc cfg_kernel = DeepKernelConfig(**cfg_kernel).dict() # pydantic validation assert cfg_kernel['proj']['src'] == 'model' assert cfg_kernel['proj']['custom_objects'] is None + assert cfg_kernel['proj']['layer'] is None # Resolve and load config cfg = {'kernel': cfg_kernel, 'backend': backend} @@ -1115,10 +1119,10 @@ def test_save_preprocess(data, preprocess_fn, tmp_path, backend): # Save preprocess_fn to config filepath = tmp_path X_ref, X_h0 = data - input_dim = X_ref.shape[1] + input_shape = (X_ref.shape[1],) cfg_preprocess = _save_preprocess_config(preprocess_fn, backend=backend, - input_shape=input_dim, + input_shape=input_shape, filepath=filepath) cfg_preprocess = _path2str(cfg_preprocess) cfg_preprocess = PreprocessConfig(**cfg_preprocess).dict() # pydantic validation @@ -1136,7 +1140,7 @@ def test_save_preprocess(data, preprocess_fn, tmp_path, backend): @parametrize('preprocess_fn', [preprocess_nlp]) @parametrize_with_cases("data", cases=TextData.movie_sentiment_data, prefix='data_') -def test_save_preprocess_nlp(data, preprocess_fn, max_len, tmp_path, backend): +def test_save_preprocess_nlp(data, preprocess_fn, tmp_path, backend): """ Unit test for _save_preprocess_config and _load_preprocess_config, with text data. @@ -1147,7 +1151,7 @@ def test_save_preprocess_nlp(data, preprocess_fn, max_len, tmp_path, backend): filepath = tmp_path cfg_preprocess = _save_preprocess_config(preprocess_fn, backend=backend, - input_shape=max_len, + input_shape=(768,), # hardcoded to bert-base-cased for now filepath=filepath) cfg_preprocess = _path2str(cfg_preprocess) cfg_preprocess = PreprocessConfig(**cfg_preprocess).dict() # pydantic validation @@ -1185,8 +1189,8 @@ def test_save_model(data, model, layer, backend, tmp_path): """ # Save model filepath = tmp_path - input_dim = data[0].shape[1] - cfg_model, _ = _save_model_config(model, base_path=filepath, input_shape=input_dim, backend=backend) + input_shape = (data[0].shape[1],) + cfg_model, _ = _save_model_config(model, base_path=filepath, input_shape=input_shape, backend=backend) cfg_model = _path2str(cfg_model) cfg_model = ModelConfig(**cfg_model).dict() assert tmp_path.joinpath('model').is_dir() diff --git a/alibi_detect/tests/test_dep_management.py b/alibi_detect/tests/test_dep_management.py index bb1bbc494..e6f3fde89 100644 --- a/alibi_detect/tests/test_dep_management.py +++ b/alibi_detect/tests/test_dep_management.py @@ -190,7 +190,7 @@ def test_fetching_utils_dependencies(opt_dep): def test_saving_tf_dependencies(opt_dep): - """Tests that the alibi_detect.saving.tensorflow module correctly protects against uninstalled optional + """Tests that the alibi_detect.saving._tensorflow module correctly protects against uninstalled optional dependencies. """ @@ -208,7 +208,7 @@ def test_saving_tf_dependencies(opt_dep): ('get_tf_dtype', ['tensorflow']) ]: dependency_map[dependency] = relations - from alibi_detect.saving import tensorflow as tf_saving + from alibi_detect.saving import _tensorflow as tf_saving check_correct_dependencies(tf_saving, dependency_map, opt_dep) diff --git a/doc/source/overview/saving.md b/doc/source/overview/saving.md index 07a075e59..1e651f145 100644 --- a/doc/source/overview/saving.md +++ b/doc/source/overview/saving.md @@ -98,5 +98,42 @@ for the remaining detectors is in the [Roadmap](roadmap.md). ```` ```{note} -Saving/loading of detectors using PyTorch models and/or a PyTorch backend is currently not supported. +For detectors with backends, or using preprocessing, save/load support is currently limited to TensorFlow models and backends. ``` + +(supported_models)= +## Supported ML models + +Alibi Detect drift detectors offer the option to perform [preprocessing](../cd/background.md#input-preprocessing) +with user-defined machine learning models: + +```python +model = ... # TensorFlow model; tf.keras.Model or tf.keras.Sequential +preprocess_fn = partial(preprocess_drift, model=model, batch_size=128) +cd = MMDDrift(x_ref, backend='tensorflow', p_val=.05, preprocess_fn=preprocess_fn) +``` + +Additionally, some detectors are built upon models directly, +for example the [Classifier](../cd/methods/classifierdrift.ipynb) drift detector requires a `model` to be passed +as an argument: + +```python +cd = ClassifierDrift(x_ref, model, p_val=.05, preds_type='probs') +``` + +In order for a detector to be saveable and loadable, any models contained within it (or referenced within a +[detector configuration file](config_files.md#specifying-artefacts)) must fall within the family of supported models +documented below. + +### TensorFlow models + +Alibi Detect supports any TensorFlow model that can be serialized to the +[HDF5](https://www.tensorflow.org/guide/keras/save_and_serialize#keras_h5_format) format. +Custom objects should be pre-registered with +[register_keras_serializable](https://www.tensorflow.org/api_docs/python/tf/keras/utils/register_keras_serializable). +``` + +%### PyTorch + +%### scikit-learn + From a8a8e8cacea70648f661741e027a9d11bad0c3b1 Mon Sep 17 00:00:00 2001 From: mauicv Date: Thu, 29 Sep 2022 10:39:15 +0100 Subject: [PATCH 24/35] Remove has_tensorflow checks from cd init methods (#632) --- alibi_detect/cd/context_aware.py | 2 +- alibi_detect/cd/lsdd.py | 2 +- alibi_detect/cd/lsdd_online.py | 2 +- alibi_detect/cd/mmd_online.py | 2 +- alibi_detect/cd/spot_the_diff.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/alibi_detect/cd/context_aware.py b/alibi_detect/cd/context_aware.py index 4f0093481..6bc455c4b 100644 --- a/alibi_detect/cd/context_aware.py +++ b/alibi_detect/cd/context_aware.py @@ -113,7 +113,7 @@ def __init__( if c_kernel is None: kwargs.update({'c_kernel': GaussianRBF}) - if backend == 'tensorflow' and has_tensorflow: + if backend == 'tensorflow': kwargs.pop('device', None) self._detector = ContextMMDDriftTF(*args, **kwargs) # type: ignore else: diff --git a/alibi_detect/cd/lsdd.py b/alibi_detect/cd/lsdd.py index d7264898b..8335dd3a2 100644 --- a/alibi_detect/cd/lsdd.py +++ b/alibi_detect/cd/lsdd.py @@ -92,7 +92,7 @@ def __init__( pop_kwargs = ['self', 'x_ref', 'backend', '__class__'] [kwargs.pop(k, None) for k in pop_kwargs] - if backend == 'tensorflow' and has_tensorflow: + if backend == 'tensorflow': kwargs.pop('device', None) self._detector = LSDDDriftTF(*args, **kwargs) # type: ignore else: diff --git a/alibi_detect/cd/lsdd_online.py b/alibi_detect/cd/lsdd_online.py index 57a38b3b9..45c551d17 100644 --- a/alibi_detect/cd/lsdd_online.py +++ b/alibi_detect/cd/lsdd_online.py @@ -93,7 +93,7 @@ def __init__( pop_kwargs = ['self', 'x_ref', 'ert', 'window_size', 'backend', '__class__'] [kwargs.pop(k, None) for k in pop_kwargs] - if backend == 'tensorflow' and has_tensorflow: + if backend == 'tensorflow': kwargs.pop('device', None) self._detector = LSDDDriftOnlineTF(*args, **kwargs) # type: ignore else: diff --git a/alibi_detect/cd/mmd_online.py b/alibi_detect/cd/mmd_online.py index 0308c5ad1..409473d43 100644 --- a/alibi_detect/cd/mmd_online.py +++ b/alibi_detect/cd/mmd_online.py @@ -93,7 +93,7 @@ def __init__( from alibi_detect.utils.pytorch.kernels import GaussianRBF # type: ignore kwargs.update({'kernel': GaussianRBF}) - if backend == 'tensorflow' and has_tensorflow: + if backend == 'tensorflow': kwargs.pop('device', None) self._detector = MMDDriftOnlineTF(*args, **kwargs) # type: ignore else: diff --git a/alibi_detect/cd/spot_the_diff.py b/alibi_detect/cd/spot_the_diff.py index d24be2fc7..d71e85ead 100644 --- a/alibi_detect/cd/spot_the_diff.py +++ b/alibi_detect/cd/spot_the_diff.py @@ -138,7 +138,7 @@ def __init__( pop_kwargs += ['optimizer'] [kwargs.pop(k, None) for k in pop_kwargs] - if backend == 'tensorflow' and has_tensorflow: + if backend == 'tensorflow': pop_kwargs = ['device', 'dataloader'] [kwargs.pop(k, None) for k in pop_kwargs] if dataset is None: From 1e53f70d50d777d526b9e8aee80523c76573579c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 29 Sep 2022 10:40:07 +0100 Subject: [PATCH 25/35] Update pytest-cov requirement from <4.0.0,>=2.6.1 to >=2.6.1,<5.0.0 (#638) Updates the requirements on [pytest-cov](https://github.com/pytest-dev/pytest-cov) to permit the latest version. - [Release notes](https://github.com/pytest-dev/pytest-cov/releases) - [Changelog](https://github.com/pytest-dev/pytest-cov/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-cov/compare/v2.6.1...v4.0.0) --- updated-dependencies: - dependency-name: pytest-cov dependency-type: direct:development ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 587b4363c..f34e27622 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -6,7 +6,7 @@ types-requests>=2.25, <3.0 types-toml>=0.10, <1.0 # testing pytest>=5.3.5, <8.0.0 -pytest-cov>=2.6.1, <4.0.0 +pytest-cov>=2.6.1, <5.0.0 pytest-xdist>=1.28.0, <3.0.0 # for distributed testing, currently unused (see setup.cfg) pytest_cases>=3.6.8, <4.0.0 pytest-randomly>=3.5.0, <4.0.0 From 59082b90559807b942b81848d01504b19c8a47a7 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Thu, 29 Sep 2022 11:16:38 +0100 Subject: [PATCH 26/35] Fix error in save/load table (#640) --- doc/source/overview/saving.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/overview/saving.md b/doc/source/overview/saving.md index 1e651f145..19078652a 100644 --- a/doc/source/overview/saving.md +++ b/doc/source/overview/saving.md @@ -56,8 +56,8 @@ for the remaining detectors is in the [Roadmap](roadmap.md). | Detector | Legacy save/load | Config save/load | |:-------------------------------------------------------------------------------|:----------------:|:----------------:| | [Kolmogorov-Smirnov](../cd/methods/ksdrift.ipynb) | ✅ | ✅ | -| [Cramér-von Mises](../cd/methods/cvmdrift.ipynb) | ✅ | ✅ | -| [Fisher's Exact Test](../cd/methods/fetdrift.ipynb) | ✅ | ✅ | +| [Cramér-von Mises](../cd/methods/cvmdrift.ipynb) | ❌ | ✅ | +| [Fisher's Exact Test](../cd/methods/fetdrift.ipynb) | ❌ | ✅ | | [Least-Squares Density Difference](../cd/methods/lsdddrift.ipynb) | ❌ | ✅ | | [Maximum Mean Discrepancy](../cd/methods/mmddrift.ipynb) | ✅ | ✅ | | [Learned Kernel MMD](../cd/methods/learnedkerneldrift.ipynb) | ❌ | ✅ | From 0bf6de9860fcd1b7a016e3152e6805ed74e6f15b Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Thu, 29 Sep 2022 11:30:33 +0100 Subject: [PATCH 27/35] Remove pin to 3.10.6 (#639) --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f242eb2d..77ad50d71 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,12 +30,12 @@ jobs: strategy: matrix: os: [ ubuntu-latest ] - python-version: [ '3.7', '3.8', '3.9', '3.10.6' ] + python-version: [ '3.7', '3.8', '3.9', '3.10' ] include: # Run macos and windows tests on only one python version - os: windows-latest python-version: '3.9' # PyTorch doesn't yet have 3.10 support on Windows (https://pytorch.org/get-started/locally/#windows-python) - os: macos-latest - python-version: '3.10.6' + python-version: '3.10' steps: - name: Checkout code From c6e78fcc28652f54fb71fce46e2ad189c907b645 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Fri, 30 Sep 2022 15:22:01 +0100 Subject: [PATCH 28/35] Remove config_spec versioning (#641) --- CHANGELOG.md | 2 ++ alibi_detect/base.py | 3 +-- alibi_detect/saving/schemas.py | 1 - alibi_detect/saving/tests/test_validate.py | 12 ++---------- alibi_detect/saving/validate.py | 10 +--------- alibi_detect/version.py | 6 ------ doc/source/overview/config_files.md | 3 --- 7 files changed, 6 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ebcc88b6..2cbb1101d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ See the [documentation](https://docs.seldon.io/projects/alibi-detect/en/latest/c ### Changed - Minimum `prophet` version bumped to `1.1.0` (used by `OutlierProphet`). This upgrade removes the dependency on `pystan` as `cmdstanpy` is used instead. This version also comes with pre-built wheels for all major platforms and Python versions, making both installation and testing easier ([#627](https://github.com/SeldonIO/alibi-detect/pull/627)). +- **Breaking change** The configuration field `config_spec` has been removed. In order to load detectors serialized from previous Alibi Detect versions, the field will need to be deleted from the detector's `config.toml` file. However, in any case, serialization compatibility across Alibi Detect versions is not currently guranteed. ([#641](https://github.com/SeldonIO/alibi-detect/pull/641)). + ### Development - UTF-8 decoding is enforced when `README.md` is opened by `setup.py`. This is to prevent pip install errors on systems with `PYTHONIOENCODING` set to use other encoders ([#605](https://github.com/SeldonIO/alibi-detect/pull/605)). diff --git a/alibi_detect/base.py b/alibi_detect/base.py index 847220f03..aae6896eb 100644 --- a/alibi_detect/base.py +++ b/alibi_detect/base.py @@ -4,7 +4,7 @@ import numpy as np from typing import Dict, Any, Optional from typing_extensions import Protocol, runtime_checkable -from alibi_detect.version import __version__, __config_spec__ +from alibi_detect.version import __version__ DEFAULT_META = { @@ -173,7 +173,6 @@ def _set_config(self, inputs): # TODO - move to BaseDetector once config save/l 'name': name, 'meta': { 'version': __version__, - 'config_spec': __config_spec__, } } diff --git a/alibi_detect/saving/schemas.py b/alibi_detect/saving/schemas.py index 196c8e121..e151c0bce 100644 --- a/alibi_detect/saving/schemas.py +++ b/alibi_detect/saving/schemas.py @@ -88,7 +88,6 @@ class Config: class MetaData(CustomBaseModel): version: str - config_spec: str version_warning: bool = False diff --git a/alibi_detect/saving/tests/test_validate.py b/alibi_detect/saving/tests/test_validate.py index 19780d69f..10871f4ff 100644 --- a/alibi_detect/saving/tests/test_validate.py +++ b/alibi_detect/saving/tests/test_validate.py @@ -4,14 +4,13 @@ from alibi_detect.saving import validate_config from alibi_detect.saving.saving import X_REF_FILENAME -from alibi_detect.version import __config_spec__, __version__ +from alibi_detect.version import __version__ from copy import deepcopy # Define a detector config dict mmd_cfg = { 'meta': { 'version': __version__, - 'config_spec': __config_spec__, }, 'name': 'MMDDrift', 'x_ref': np.array([[-0.30074928], [1.50240758], [0.43135768], [2.11295779], [0.79684913]]), @@ -32,7 +31,6 @@ def test_validate_config(cfg): # Check cfg is returned with correct metadata meta = cfg_full.get('meta') # pop as don't want to compare meta to cfg in next bit assert meta['version'] == __version__ - assert meta['config_spec'] == __config_spec__ assert not meta.pop('version_warning') # pop this one to remove from next check # Check remaining values of items in cfg unchanged @@ -45,19 +43,13 @@ def test_validate_config(cfg): _ = validate_config(cfg_unres) assert not cfg.get('meta').get('version_warning') - # Check warning raised and warning field added if version or config_spec different + # Check warning raised and warning field added if version different cfg_err = cfg.copy() cfg_err['meta']['version'] = '0.1.x' with pytest.warns(Warning): # error will be raised if a warning IS NOT raised cfg_err = validate_config(cfg_err, resolved=True) assert cfg_err.get('meta').get('version_warning') - cfg_err = cfg.copy() - cfg_err['meta']['config_spec'] = '0.x' - with pytest.warns(Warning): # error will be raised if a warning IS NOT raised - cfg_err = validate_config(cfg_err, resolved=True) - assert cfg_err.get('meta').get('version_warning') - # Check ValueError raised if name unrecognised cfg_err = cfg.copy() cfg_err['name'] = 'MMDDriftWrong' diff --git a/alibi_detect/saving/validate.py b/alibi_detect/saving/validate.py index 235a74965..672ee7431 100644 --- a/alibi_detect/saving/validate.py +++ b/alibi_detect/saving/validate.py @@ -2,7 +2,7 @@ from alibi_detect.saving.schemas import ( # type: ignore[attr-defined] DETECTOR_CONFIGS, DETECTOR_CONFIGS_RESOLVED) -from alibi_detect.version import __config_spec__, __version__ +from alibi_detect.version import __version__ def validate_config(cfg: dict, resolved: bool = False) -> dict: @@ -41,7 +41,6 @@ def validate_config(cfg: dict, resolved: bool = False) -> dict: meta = {} if meta is None else meta # Needed because pydantic sets meta=None if it is missing from the config version_warning = meta.get('version_warning', False) version = meta.get('version', None) - config_spec = meta.get('config_spec', None) # Raise warning if config file already contains a version_warning if version_warning: @@ -54,11 +53,4 @@ def validate_config(cfg: dict, resolved: bool = False) -> dict: f'{__version__}. This may lead to breaking code or invalid results.') cfg['meta'].update({'version_warning': True}) - # Check config specification version - if config_spec is not None and config_spec != __config_spec__: - warnings.warn(f'Config has specification {version} when the installed ' - f'alibi-detect version expects specification {__config_spec__}.' - 'This may lead to breaking code or invalid results.') - cfg['meta'].update({'version_warning': True}) - return cfg diff --git a/alibi_detect/version.py b/alibi_detect/version.py index 028aea0b5..82c9db9a1 100644 --- a/alibi_detect/version.py +++ b/alibi_detect/version.py @@ -3,9 +3,3 @@ # 2) we can import it in setup.py for the same reason # 3) we can import it into your module module __version__ = "0.10.4dev" - -# Define the config specification version. This is distinct to the library version above. It is only updated when -# any detector config schema is updated, such that loading a previous config spec cannot be guaranteed to work. -# The minor version number is associated with minor changes such as adding/removing/changing kwarg's, whilst the major -# number is reserved for significant changes to the config layout. -__config_spec__ = "0.1" diff --git a/doc/source/overview/config_files.md b/doc/source/overview/config_files.md index 3802524c1..669e60c76 100644 --- a/doc/source/overview/config_files.md +++ b/doc/source/overview/config_files.md @@ -461,9 +461,6 @@ artefacts before attempting the sometimes time-consuming operation of instantiat %```python %{'name': {'title': 'Name', 'type': 'string'}, % 'version': {'title': 'Version', 'default': '0.8.1dev', 'type': 'string'}, -% 'config_spec': {'title': 'Config Spec', -% 'default': '0.1.0dev', -% 'type': 'string'}, % 'backend': {'title': 'Backend', % 'default': 'tensorflow', % 'enum': ['tensorflow', 'pytorch'], From 8f42ee256ce088bd7362e25177ebac1c1f8d8ab4 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Mon, 3 Oct 2022 12:27:02 +0100 Subject: [PATCH 29/35] Add option to ssh into GitHub Actions for debugging (#644) --- .github/workflows/ci.yml | 12 ++++++++++++ CONTRIBUTING.md | 32 +++++++++++++++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77ad50d71..a7de840f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,12 @@ on: - cron: '0 0 * * *' # Trigger the workflow on manual dispatch workflow_dispatch: + inputs: + tmate_enabled: + type: boolean + description: 'Enable tmate debugging?' + required: false + default: false jobs: @@ -57,6 +63,12 @@ jobs: python -m pip install --upgrade --upgrade-strategy eager -e .[prophet,tensorflow,torch] python -m pip freeze + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + if: ${{ github.event_name == 'workflow_dispatch' && inputs.tmate_enabled }} + with: + limit-access-to-actor: true + - name: Lint with flake8 run: | flake8 alibi_detect diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7c8c3e7d0..a10e54b53 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,7 +50,37 @@ We use `sphinx` for building documentation. You can call `make build_docs` from the docs will be built under `doc/_build/html`. ## CI -All PRs triger a Github Actions build to run linting, type checking, tests, and build docs. +All PRs triger a Github Actions build to run linting, type checking, tests, and build docs. The status of each +Github Action can be viewed on the [actions page](https://github.com/SeldonIO/alibi-detect/actions). + +### Debugging via CI + +For various reasons, CI runs might occasionally fail. They can often be debugged locally, but sometimes it is helpful +to debug them in the exact enviroment seen during CI. For this purpose, there is the facilty to ssh directly into +the CI Guthub Action runner. + +#### Instructions + +1. Go to the "CI" workflows section on the Alibi Detect GitHub Actions page. + +2. Click on "Run Workflow", and select the "Enable tmate debugging" toggle. + +3. Select the workflow once it starts, and then select the build of interest (e.g. `ubuntu-latest, 3.10`). + +4. Once the workflow reaches the `Setup tmate session` step, click on the toggle to expand it. + +5. Copy and paste the `ssh` command that is being printed to your terminal e.g. `ssh TaHAR65KFbjbSdrkwxNz996TL@nyc1.tmate.io`. + +6. Run the ssh command locally. Assuming your ssh keys are properly set up for github, you should now be inside the GutHub Action runner. + +7. The tmate session is opened after the Python and pip installs are completed, so you should be ready to run `alibi-detect` and debug as required. + +#### Additional notes + +- If the registered public SSH key is not your default private SSH key, you will need to specify the path manually, like so: ssh -i . +- Once you have finished debugging, you can continue the workflow (i.e. let the full build CI run) by running `touch continue` whilst in the root directory (`~/work/alibi-detect/alibi-detect`). This will close the tmate session. +- This new capability is currently temperamental on the `MacOS` build due to [this issue](https://github.com/mxschmitt/action-tmate/issues/69). If the MacOS build fails all the builds are failed. If this happens, it is +recommended to retrigger only the workflow build of interest e.g. `ubuntu-latest, 3.10`, and then follow the instructions above from step 3. ## Optional Dependencies From 5b14ab0d3f8414853d7eded87ab19a348f55159b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Oct 2022 17:12:47 +0100 Subject: [PATCH 30/35] Update tensorflow requirement (#608) Updates the requirements on [tensorflow](https://github.com/tensorflow/tensorflow) to permit the latest version. - [Release notes](https://github.com/tensorflow/tensorflow/releases) - [Changelog](https://github.com/tensorflow/tensorflow/blob/master/RELEASE.md) - [Commits](https://github.com/tensorflow/tensorflow/compare/v2.2.0...v2.10.0) --- updated-dependencies: - dependency-name: tensorflow dependency-type: direct:development ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 080ca56b4..06f14663b 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ def readme(): # https://github.com/SeldonIO/alibi-detect/issues/375 and 387 "tensorflow": [ "tensorflow_probability>=0.8.0, <0.18.0", - "tensorflow>=2.2.0, !=2.6.0, !=2.6.1, <2.10.0", # https://github.com/SeldonIO/alibi-detect/issues/375 and 387 + "tensorflow>=2.2.0, !=2.6.0, !=2.6.1, <2.11.0", # https://github.com/SeldonIO/alibi-detect/issues/375 and 387 ], "keops": [ "pykeops>=2.0.0, <2.2.0", @@ -28,7 +28,7 @@ def readme(): "all": [ "prophet>=1.1.0, <2.0.0", "tensorflow_probability>=0.8.0, <0.18.0", - "tensorflow>=2.2.0, !=2.6.0, !=2.6.1, <2.10.0", # https://github.com/SeldonIO/alibi-detect/issues/375 and 387 + "tensorflow>=2.2.0, !=2.6.0, !=2.6.1, <2.11.0", # https://github.com/SeldonIO/alibi-detect/issues/375 and 387 "pykeops>=2.0.0, <2.2.0", "torch>=1.7.0, <1.13.0" ], From 6fa17081735c11275dc7e551ddc672674e679b93 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Oct 2022 18:16:21 +0100 Subject: [PATCH 31/35] Update tensorflow-probability requirement (#616) Updates the requirements on [tensorflow-probability](https://github.com/tensorflow/probability) to permit the latest version. - [Release notes](https://github.com/tensorflow/probability/releases) - [Commits](https://github.com/tensorflow/probability/compare/0.8.0...v0.18.0) --- updated-dependencies: - dependency-name: tensorflow-probability dependency-type: direct:development ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 06f14663b..2324a2ce7 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def readme(): ], # https://github.com/SeldonIO/alibi-detect/issues/375 and 387 "tensorflow": [ - "tensorflow_probability>=0.8.0, <0.18.0", + "tensorflow_probability>=0.8.0, <0.19.0", "tensorflow>=2.2.0, !=2.6.0, !=2.6.1, <2.11.0", # https://github.com/SeldonIO/alibi-detect/issues/375 and 387 ], "keops": [ @@ -27,7 +27,7 @@ def readme(): ], "all": [ "prophet>=1.1.0, <2.0.0", - "tensorflow_probability>=0.8.0, <0.18.0", + "tensorflow_probability>=0.8.0, <0.19.0", "tensorflow>=2.2.0, !=2.6.0, !=2.6.1, <2.11.0", # https://github.com/SeldonIO/alibi-detect/issues/375 and 387 "pykeops>=2.0.0, <2.2.0", "torch>=1.7.0, <1.13.0" From b915d633120395ae0b647369f0637fc527073047 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Oct 2022 15:15:19 +0100 Subject: [PATCH 32/35] Update numba requirement from !=0.54.0,<0.56.0,>=0.50.0 to >=0.50.0,!=0.54.0,<0.57.0 (#573) * Update numba requirement Updates the requirements on [numba](https://github.com/numba/numba) to permit the latest version. - [Release notes](https://github.com/numba/numba/releases) - [Commits](https://github.com/numba/numba/compare/0.50.0...0.56.0) --- updated-dependencies: - dependency-name: numba dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ashley Scillitoe --- .github/workflows/ci.yml | 3 +++ doc/source/cd/methods/onlinecvmdrift.ipynb | 12 ++++++++++++ setup.py | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7de840f4..e344f2395 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,6 +79,9 @@ jobs: - name: Test with pytest run: | + if [ "$RUNNER_OS" == "macOS" ]; then # Avoid numba/OpenMP segfault in CVMDrift (https://github.com/SeldonIO/alibi-detect/issues/648) + export NUMBA_THREADING_LAYER="workqueue" + fi pytest alibi_detect - name: Upload coverage to Codecov diff --git a/doc/source/cd/methods/onlinecvmdrift.ipynb b/doc/source/cd/methods/onlinecvmdrift.ipynb index a5bfd80e8..71e670087 100644 --- a/doc/source/cd/methods/onlinecvmdrift.ipynb +++ b/doc/source/cd/methods/onlinecvmdrift.ipynb @@ -17,6 +17,18 @@ "\n", "The online [Cramér-von Mises](https://en.wikipedia.org/wiki/Cram%C3%A9r%E2%80%93von_Mises_criterion) detector is a non-parametric method for online drift detection on continuous data. Like the [offline Cramér-von Mises](cvmdrift.ipynb) detector, it applies a univariate Cramér-von Mises (CVM) test to each feature. This detector is an adaptation of that proposed in [this paper](https://www.tandfonline.com/doi/abs/10.1080/00224065.2012.11917887) by Ross et al. .\n", "\n", + "
\n", + "\n", + "**Warning**\n", + " \n", + "This detector is multi-threaded, with Numba used to parallelise over the simulated streams. There is a [known issue](https://github.com/SeldonIO/alibi-detect/issues/648) on MacOS, where Numba's default OpenMP [threading layer](https://numba.readthedocs.io/en/stable/user/threading-layer.html?highlight=thread_id#the-threading-layers) causes segfaults. A workaround is to use the slightly less performant `workqueue` threading layer on MacOS by setting the `NUMBA_THREADING_LAYER` enviroment variable or running:\n", + " \n", + "```python\n", + "from numba import config\n", + "config.THREADING_LAYER = 'workqueue'\n", + "```\n", + "
\n", + "\n", "### Threshold configuration\n", "Online detectors assume the reference data is large and fixed and operate on single data points at a time (rather than batches). These data points are passed into the test-windows, and a two-sample test-statistic between the reference data and test-window is computed at each time-step. When the test-statistic exceeds a preconfigured threshold, drift is detected. Configuration of the thresholds requires specification of the expected run-time (ERT) which specifies how many time-steps that the detector, on average, should run for in the absence of drift before making a false detection. Thresholds are then configured to target this ERT by simulating `n_bootstraps` number of streams of length `t_max = 2*max(window_sizes) - 1`. Conveniently, the non-parametric nature of the detector means that thresholds depend only on $M$, the length of the reference data set. Therefore, for multivariate data, configuration is only as costly as the univariate case.\n", "\n", diff --git a/setup.py b/setup.py index 2324a2ce7..1385d3368 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ def readme(): "pydantic>=1.8.0, <2.0.0", "toml>=0.10.1, <1.0.0", # STC, see https://discuss.python.org/t/adopting-recommending-a-toml-parser/4068 "catalogue>=2.0.0, <3.0.0", - "numba>=0.50.0, !=0.54.0, <0.56.0", # Avoid 0.54 due to: https://github.com/SeldonIO/alibi/issues/466 + "numba>=0.50.0, !=0.54.0, <0.57.0", # Avoid 0.54 due to: https://github.com/SeldonIO/alibi/issues/466 "typing-extensions>=3.7.4.3" ], extras_require=extras_require, From 1898ad2d451180c5c715f0834a9e2ea3005e3619 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Wed, 12 Oct 2022 16:22:27 +0100 Subject: [PATCH 33/35] Support for serializing detectors with scikit-learn backends and/or models (#642) --- alibi_detect/saving/_sklearn/__init__.py | 7 + alibi_detect/saving/_sklearn/loading.py | 26 ++ alibi_detect/saving/_sklearn/saving.py | 68 ++++ .../saving/_sklearn/tests/test_saving_sk.py | 32 ++ alibi_detect/saving/_tensorflow/loading.py | 22 +- alibi_detect/saving/_tensorflow/saving.py | 13 +- .../_tensorflow/tests/test_saving_tf.py | 64 +++ alibi_detect/saving/loading.py | 53 +-- alibi_detect/saving/saving.py | 41 +- alibi_detect/saving/schemas.py | 97 +++-- alibi_detect/saving/tests/models.py | 251 ++++++++++++ alibi_detect/saving/tests/test_saving.py | 376 ++++-------------- doc/source/overview/saving.md | 12 +- requirements/dev.txt | 1 + 14 files changed, 660 insertions(+), 403 deletions(-) create mode 100644 alibi_detect/saving/_sklearn/__init__.py create mode 100644 alibi_detect/saving/_sklearn/loading.py create mode 100644 alibi_detect/saving/_sklearn/saving.py create mode 100644 alibi_detect/saving/_sklearn/tests/test_saving_sk.py create mode 100644 alibi_detect/saving/_tensorflow/tests/test_saving_tf.py create mode 100644 alibi_detect/saving/tests/models.py diff --git a/alibi_detect/saving/_sklearn/__init__.py b/alibi_detect/saving/_sklearn/__init__.py new file mode 100644 index 000000000..59b424eb6 --- /dev/null +++ b/alibi_detect/saving/_sklearn/__init__.py @@ -0,0 +1,7 @@ +from alibi_detect.saving._sklearn.saving import save_model_config as save_model_config_sk +from alibi_detect.saving._sklearn.loading import load_model as load_model_sk + +__all__ = [ + "save_model_config_sk", + "load_model_sk" +] diff --git a/alibi_detect/saving/_sklearn/loading.py b/alibi_detect/saving/_sklearn/loading.py new file mode 100644 index 000000000..b6ed23912 --- /dev/null +++ b/alibi_detect/saving/_sklearn/loading.py @@ -0,0 +1,26 @@ +import os +from pathlib import Path +from typing import Union + +import joblib +from sklearn.base import BaseEstimator + + +def load_model(filepath: Union[str, os.PathLike], + ) -> BaseEstimator: + """ + Load scikit-learn (or xgboost) model. Models are assumed to be a subclass of :class:`~sklearn.base.BaseEstimator`. + This includes xgboost models following the scikit-learn API + (see https://xgboost.readthedocs.io/en/latest/python/python_api.html#module-xgboost.sklearn). + + Parameters + ---------- + filepath + Saved model directory. + + Returns + ------- + Loaded model. + """ + model_dir = Path(filepath) + return joblib.load(model_dir.joinpath('model.joblib')) diff --git a/alibi_detect/saving/_sklearn/saving.py b/alibi_detect/saving/_sklearn/saving.py new file mode 100644 index 000000000..bbace2388 --- /dev/null +++ b/alibi_detect/saving/_sklearn/saving.py @@ -0,0 +1,68 @@ +import logging +import os +from pathlib import Path +from typing import Union + +import joblib +from sklearn.base import BaseEstimator + +logger = logging.getLogger(__name__) + + +def save_model_config(model: BaseEstimator, + base_path: Path, + local_path: Path = Path('.')) -> dict: + """ + Save a scikit-learn (or xgboost) model to a config dictionary. + Models are assumed to be a subclass of :class:`~sklearn.base.BaseEstimator`. This includes xgboost models + following the scikit-learn API + (see https://xgboost.readthedocs.io/en/latest/python/python_api.html#module-xgboost.sklearn). + + Parameters + ---------- + model + The model to save. + base_path + Base filepath to save to (the location of the `config.toml` file). + local_path + A local (relative) filepath to append to base_path. + + Returns + ------- + The model config dict. + """ + filepath = base_path.joinpath(local_path) + save_model(model, filepath=filepath, save_dir='model') + cfg_model = { + 'flavour': 'sklearn', + 'src': local_path.joinpath('model') + } + return cfg_model + + +def save_model(model: BaseEstimator, + filepath: Union[str, os.PathLike], + save_dir: Union[str, os.PathLike] = 'model') -> None: + """ + Save scikit-learn (and xgboost) models. Models are assumed to be a subclass of :class:`~sklearn.base.BaseEstimator`. + This includes xgboost models following the scikit-learn API + (see https://xgboost.readthedocs.io/en/latest/python/python_api.html#module-xgboost.sklearn). + + Parameters + ---------- + model + The tf.keras.Model to save. + filepath + Save directory. + save_dir + Name of folder to save to within the filepath directory. + """ + # create folder to save model in + model_path = Path(filepath).joinpath(save_dir) + if not model_path.is_dir(): + logger.warning('Directory {} does not exist and is now created.'.format(model_path)) + model_path.mkdir(parents=True, exist_ok=True) + + # save model + model_path = model_path.joinpath('model.joblib') + joblib.dump(model, model_path) diff --git a/alibi_detect/saving/_sklearn/tests/test_saving_sk.py b/alibi_detect/saving/_sklearn/tests/test_saving_sk.py new file mode 100644 index 000000000..3bc588553 --- /dev/null +++ b/alibi_detect/saving/_sklearn/tests/test_saving_sk.py @@ -0,0 +1,32 @@ +from pytest_cases import param_fixture, parametrize, parametrize_with_cases + +from alibi_detect.saving.tests.datasets import ContinuousData +from alibi_detect.saving.tests.models import classifier_model, xgb_classifier_model + +from alibi_detect.saving.loading import _load_model_config +from alibi_detect.saving.saving import _path2str, _save_model_config +from alibi_detect.saving.schemas import ModelConfig + +backend = param_fixture("backend", ['sklearn']) + + +@parametrize_with_cases("data", cases=ContinuousData.data_synthetic_nd, prefix='data_') +@parametrize('model', [classifier_model, xgb_classifier_model]) +def test_save_model_sk(data, model, tmp_path): + """ + Unit test for _save_model_config and _load_model_config with scikit-learn and xgboost model. + """ + # Save model + filepath = tmp_path + cfg_model, _ = _save_model_config(model, base_path=filepath) + cfg_model = _path2str(cfg_model) + cfg_model = ModelConfig(**cfg_model).dict() + assert tmp_path.joinpath('model').is_dir() + assert tmp_path.joinpath('model/model.joblib').is_file() + + # Adjust config + cfg_model['src'] = tmp_path.joinpath('model') # Need to manually set to absolute path here + + # Load model + model_load = _load_model_config(cfg_model) + assert isinstance(model_load, type(model)) diff --git a/alibi_detect/saving/_tensorflow/loading.py b/alibi_detect/saving/_tensorflow/loading.py index 1ce7c43f6..ec38b0996 100644 --- a/alibi_detect/saving/_tensorflow/loading.py +++ b/alibi_detect/saving/_tensorflow/loading.py @@ -69,7 +69,7 @@ def load_model(filepath: Union[str, os.PathLike], return model -def prep_model_and_emb(model: Optional[Callable], emb: Optional[TransformerEmbedding]) -> Callable: +def prep_model_and_emb(model: Callable, emb: Optional[TransformerEmbedding]) -> Callable: """ Function to perform final preprocessing of model (and/or embedding) before it is passed to preprocess_drift. @@ -78,25 +78,17 @@ def prep_model_and_emb(model: Optional[Callable], emb: Optional[TransformerEmbed model A compatible model. emb - A text embedding model. + An optional text embedding model. Returns ------- The final model ready to passed to preprocess_drift. """ - # If a model exists, process it (and embedding) - if model is not None: - model = model.encoder if isinstance(model, UAE) else model # This is to avoid nesting UAE's already a UAE - if emb is not None: - model = _Encoder(emb, mlp=model) - model = UAE(encoder_net=model) - # If no model exists, store embedding as model - else: - model = emb - if model is None: - raise ValueError("A 'model' and/or `embedding` must be specified when " - "preprocess_fn='preprocess_drift'") - + # Process model (and embedding) + model = model.encoder if isinstance(model, UAE) else model # This is to avoid nesting UAE's already a UAE + if emb is not None: + model = _Encoder(emb, mlp=model) + model = UAE(encoder_net=model) return model diff --git a/alibi_detect/saving/_tensorflow/saving.py b/alibi_detect/saving/_tensorflow/saving.py index 1b4ef7126..9705d62de 100644 --- a/alibi_detect/saving/_tensorflow/saving.py +++ b/alibi_detect/saving/_tensorflow/saving.py @@ -28,10 +28,10 @@ def save_model_config(model: Callable, base_path: Path, - input_shape: tuple, + input_shape: Optional[tuple], local_path: Path = Path('.')) -> Tuple[dict, Optional[dict]]: """ - Save a model to a config dictionary. When a model has a text embedding model contained within it, + Save a TensorFlow model to a config dictionary. When a model has a text embedding model contained within it, this is extracted and saved separately. Parameters @@ -53,6 +53,9 @@ def save_model_config(model: Callable, cfg_embed = None # type: Optional[Dict[str, Any]] if isinstance(model, UAE): if isinstance(model.encoder.layers[0], TransformerEmbedding): # if UAE contains embedding and encoder + if input_shape is None: + raise ValueError('Cannot save combined embedding and model when `input_shape` is None.') + # embedding embed = model.encoder.layers[0] cfg_embed = save_embedding_config(embed, base_path, local_path.joinpath('embedding')) @@ -78,7 +81,10 @@ def save_model_config(model: Callable, if model is not None: filepath = base_path.joinpath(local_path) save_model(model, filepath=filepath, save_dir='model') - cfg_model = {'src': local_path.joinpath('model')} + cfg_model = { + 'flavour': 'tensorflow', + 'src': local_path.joinpath('model') + } return cfg_model, cfg_embed @@ -142,6 +148,7 @@ def save_embedding_config(embed: TransformerEmbedding, cfg_embed.update({'type': embed.emb_type}) cfg_embed.update({'layers': embed.hs_emb.keywords['layers']}) cfg_embed.update({'src': local_path}) + cfg_embed.update({'flavour': 'tensorflow'}) # Save embedding model logger.info('Saving embedding model to {}.'.format(filepath)) diff --git a/alibi_detect/saving/_tensorflow/tests/test_saving_tf.py b/alibi_detect/saving/_tensorflow/tests/test_saving_tf.py new file mode 100644 index 000000000..0d96e9b75 --- /dev/null +++ b/alibi_detect/saving/_tensorflow/tests/test_saving_tf.py @@ -0,0 +1,64 @@ +from pytest_cases import param_fixture, parametrize, parametrize_with_cases + +from alibi_detect.saving.tests.datasets import ContinuousData +from alibi_detect.saving.tests.models import encoder_model + +from alibi_detect.cd.tensorflow import HiddenOutput as HiddenOutput_tf +from alibi_detect.saving.loading import _load_model_config, _load_optimizer_config +from alibi_detect.saving.saving import _path2str, _save_model_config +from alibi_detect.saving.schemas import ModelConfig + +backend = param_fixture("backend", ['tensorflow']) + + +def test_load_optimizer_tf(backend): + "Test the tensorflow _load_optimizer_config." + class_name = 'Adam' + learning_rate = 0.01 + epsilon = 1e-7 + amsgrad = False + + # Load + cfg_opt = { + 'class_name': class_name, + 'config': { + 'name': class_name, + 'learning_rate': learning_rate, + 'epsilon': epsilon, + 'amsgrad': amsgrad + } + } + optimizer = _load_optimizer_config(cfg_opt, backend=backend) + assert type(optimizer).__name__ == class_name + assert optimizer.learning_rate == learning_rate + assert optimizer.epsilon == epsilon + assert optimizer.amsgrad == amsgrad + + +@parametrize_with_cases("data", cases=ContinuousData.data_synthetic_nd, prefix='data_') +@parametrize('model', [encoder_model]) +@parametrize('layer', [None, -1]) +def test_save_model_tf(data, model, layer, tmp_path): + """ + Unit test for _save_model_config and _load_model_config with tensorflow model. + """ + # Save model + filepath = tmp_path + input_shape = (data[0].shape[1],) + cfg_model, _ = _save_model_config(model, base_path=filepath, input_shape=input_shape) + cfg_model = _path2str(cfg_model) + cfg_model = ModelConfig(**cfg_model).dict() + assert tmp_path.joinpath('model').is_dir() + assert tmp_path.joinpath('model/model.h5').is_file() + + # Adjust config + cfg_model['src'] = tmp_path.joinpath('model') # Need to manually set to absolute path here + if layer is not None: + cfg_model['layer'] = layer + + # Load model + model_load = _load_model_config(cfg_model) + if layer is None: + assert isinstance(model_load, type(model)) + else: + assert isinstance(model_load, HiddenOutput_tf) diff --git a/alibi_detect/saving/loading.py b/alibi_detect/saving/loading.py index 88c67e0a6..385dfcb31 100644 --- a/alibi_detect/saving/loading.py +++ b/alibi_detect/saving/loading.py @@ -1,4 +1,3 @@ -# TODO - Need to modularise torch and tensorflow imports and use. e.g. has_tensorflow and has_pytorch etc import logging import os from functools import partial @@ -14,8 +13,11 @@ from alibi_detect.saving.registry import registry from alibi_detect.saving._tensorflow import load_detector_legacy, load_embedding_tf, load_kernel_config_tf, \ load_model_tf, load_optimizer_tf, prep_model_and_emb_tf, get_tf_dtype +from alibi_detect.saving._sklearn import load_model_sk from alibi_detect.saving.validate import validate_config from alibi_detect.base import Detector, ConfigurableDetector +from alibi_detect.utils.frameworks import has_tensorflow, has_pytorch +from alibi_detect.saving.schemas import SupportedModels_tf, SupportedModels_torch if TYPE_CHECKING: import tensorflow as tf @@ -127,9 +129,9 @@ def _load_detector_config(filepath: Union[str, os.PathLike]) -> ConfigurableDete logger.info('Validated resolved config.') # Backend - backend = cfg.pop('backend') # popping so that cfg left as kwargs + `name` when passed to _init_detector - if backend.lower() != 'tensorflow': - raise NotImplementedError('Loading detectors with PyTorch, sklearn or keops backend is not yet supported.') + backend = cfg.get('backend', None) + if backend is not None and backend.lower() not in ('tensorflow', 'sklearn'): + raise NotImplementedError('Loading detectors with pytorch or keops backend is not yet supported.') # Init detector from config logger.info('Instantiating detector.') @@ -185,8 +187,7 @@ def _load_kernel_config(cfg: dict, backend: str = 'tensorflow') -> Callable: return kernel -def _load_preprocess_config(cfg: dict, - backend: Optional[str] = 'tensorflow') -> Optional[Callable]: +def _load_preprocess_config(cfg: dict) -> Optional[Callable]: """ This function builds a preprocess_fn from the preprocess dict in a detector config dict. The dict format is expected to match that generated by serialize_preprocess in alibi_detect.utils.saving (also see pydantic schema). @@ -196,8 +197,6 @@ def _load_preprocess_config(cfg: dict, ---------- cfg A preprocess_fn config dict. (see pydantic schemas). - backend - The backend. Returns ------- @@ -216,14 +215,20 @@ def _load_preprocess_config(cfg: dict, emb = kwargs.pop('embedding') # embedding passed to preprocess_drift as `model` therefore remove # Backend specifics - if backend == 'tensorflow': + if has_tensorflow and isinstance(model, SupportedModels_tf): model = prep_model_and_emb_tf(model, emb) kwargs.pop('device') - elif backend == 'pytorch': # TODO - once optional deps implemented + elif has_pytorch and isinstance(model, SupportedModels_torch): raise NotImplementedError('Loading preprocess_fn for PyTorch not yet supported.') # device = cfg['device'] # TODO - device should be set already - check # kwargs.update({'model': kwargs['model'].to(device)}) # TODO - need .to(device) here? # kwargs.update({'device': device}) + elif model is None: + kwargs.pop('device') + model = emb + if model is None: + raise ValueError("A 'model' and/or `embedding` must be specified when " + "preprocess_fn='preprocess_drift'") kwargs.update({'model': model}) else: kwargs = cfg['kwargs'] # If generic callable, kwargs is cfg['kwargs'] @@ -235,18 +240,14 @@ def _load_preprocess_config(cfg: dict, return partial(preprocess_fn, **kwargs) -def _load_model_config(cfg: dict, - backend: str) -> Callable: +def _load_model_config(cfg: dict) -> Callable: """ - Loads TensorFlow, PyTorch and scikit-learn models (currently only TensorFlow supported), from a model config - dict. + Loads supported models from a model config dict. Parameters ---------- cfg Model config dict. (see pydantic model schemas). - backend - The backend. Returns ------- @@ -254,6 +255,7 @@ def _load_model_config(cfg: dict, """ # Load model + flavour = cfg['flavour'] src = cfg['src'] custom_obj = cfg['custom_objects'] layer = cfg['layer'] @@ -262,15 +264,17 @@ def _load_model_config(cfg: dict, raise FileNotFoundError("The `src` field is not a recognised directory. It should be a directory containing " "a compatible model.") - if backend == 'tensorflow': + if flavour == 'tensorflow': model = load_model_tf(src, load_dir='.', custom_objects=custom_obj, layer=layer) + elif flavour == 'sklearn': + model = load_model_sk(src) else: - raise NotImplementedError('Loading of non-tensorflow models not currently supported') + raise NotImplementedError('Loading of PyTorch models not currently supported') return model -def _load_embedding_config(cfg: dict, backend: str) -> Callable: # TODO: Could type return more tightly +def _load_embedding_config(cfg: dict) -> Callable: # TODO: Could type return more tightly """ Load a pre-trained text embedding from an embedding config dict. @@ -278,8 +282,6 @@ def _load_embedding_config(cfg: dict, backend: str) -> Callable: # TODO: Could ---------- cfg An embedding config dict. (see the pydantic schemas). - backend - The backend. Returns ------- @@ -288,7 +290,8 @@ def _load_embedding_config(cfg: dict, backend: str) -> Callable: # TODO: Could src = cfg['src'] layers = cfg['layers'] typ = cfg['type'] - if backend == 'tensorflow': + flavour = cfg['flavour'] + if flavour == 'tensorflow': emb = load_embedding_tf(src, embedding_type=typ, layers=layers) else: raise NotImplementedError('Loading of non-tensorflow embedding models not currently supported') @@ -492,15 +495,15 @@ def resolve_config(cfg: dict, config_dir: Optional[Path]) -> dict: elif isinstance(src, dict): backend = cfg.get('backend', 'tensorflow') if key[-1] in ('model', 'proj'): - obj = _load_model_config(src, backend) + obj = _load_model_config(src) elif key[-1] == 'embedding': - obj = _load_embedding_config(src, backend) + obj = _load_embedding_config(src) elif key[-1] == 'tokenizer': obj = _load_tokenizer_config(src) elif key[-1] == 'optimizer': obj = _load_optimizer_config(src, backend) elif key[-1] == 'preprocess_fn': - obj = _load_preprocess_config(src, backend) + obj = _load_preprocess_config(src) elif key[-1] in ('kernel', 'x_kernel', 'c_kernel'): obj = _load_kernel_config(src, backend) diff --git a/alibi_detect/saving/saving.py b/alibi_detect/saving/saving.py index 80c9430dd..767ad61e9 100644 --- a/alibi_detect/saving/saving.py +++ b/alibi_detect/saving/saving.py @@ -4,7 +4,7 @@ import warnings from functools import partial from pathlib import Path -from typing import Callable, Optional, Tuple, Union +from typing import Callable, Optional, Tuple, Union, Any import dill import numpy as np import toml @@ -13,9 +13,10 @@ from alibi_detect.saving._typing import VALID_DETECTORS from alibi_detect.saving.loading import _replace, validate_config from alibi_detect.saving.registry import registry -from alibi_detect.saving.schemas import SupportedModels -from alibi_detect.saving._tensorflow import save_detector_legacy, save_model_config_tf +from alibi_detect.saving.schemas import SupportedModels, SupportedModels_tf, SupportedModels_sklearn from alibi_detect.base import Detector, ConfigurableDetector +from alibi_detect.saving._tensorflow import save_detector_legacy, save_model_config_tf +from alibi_detect.saving._sklearn import save_model_config_sk # do not extend pickle dispatch table so as not to change pickle behaviour dill.extend(use_dill=False) @@ -46,8 +47,8 @@ def save_detector( if legacy: warnings.warn('The `legacy` option will be removed in a future version.', DeprecationWarning) - if 'backend' in list(detector.meta.keys()) and detector.meta['backend'] in ['pytorch', 'sklearn', 'keops']: - raise NotImplementedError('Saving detectors with PyTorch, sklearn or keops backend is not yet supported.') + if 'backend' in list(detector.meta.keys()) and detector.meta['backend'] in ['pytorch', 'keops']: + raise NotImplementedError('Saving detectors with pytorch or keops backend is not yet supported.') # TODO: Replace .__args__ w/ typing.get_args() once Python 3.7 dropped (and remove type ignore below) detector_name = detector.__class__.__name__ @@ -123,9 +124,9 @@ def _save_detector_config(detector: ConfigurableDetector, filepath: Union[str, o File path to save serialized artefacts to. """ # Get backend, input_shape and detector_name - backend = detector.meta.get('backend', 'tensorflow') - if backend != 'tensorflow': - raise NotImplementedError("Currently, saving is only supported with backend='tensorflow'.") + backend = detector.meta.get('backend', None) + if backend not in (None, 'tensorflow', 'sklearn'): + raise NotImplementedError("Currently, saving is only supported with backend='tensorflow' and 'sklearn'.") detector_name = detector.__class__.__name__ # Process file paths @@ -157,7 +158,7 @@ def _save_detector_config(detector: ConfigurableDetector, filepath: Union[str, o preprocess_fn = cfg.get('preprocess_fn', None) if preprocess_fn is not None: logger.info('Saving the preprocess_fn function.') - preprocess_cfg = _save_preprocess_config(preprocess_fn, backend, cfg['input_shape'], filepath) + preprocess_cfg = _save_preprocess_config(preprocess_fn, cfg['input_shape'], filepath) cfg['preprocess_fn'] = preprocess_cfg # Serialize kernels @@ -167,13 +168,13 @@ def _save_detector_config(detector: ConfigurableDetector, filepath: Union[str, o cfg[kernel_str] = _save_kernel_config(kernel, filepath, Path(kernel_str)) if 'proj' in cfg[kernel_str]: # serialise proj from DeepKernel - do here as need input_shape cfg[kernel_str]['proj'], _ = _save_model_config(cfg[kernel_str]['proj'], base_path=filepath, - input_shape=cfg['input_shape'], backend=backend) + input_shape=cfg['input_shape']) # ClassifierDrift and SpotTheDiffDrift specific artefacts. # Serialize detector model model = cfg.get('model', None) if model is not None: - model_cfg, _ = _save_model_config(model, base_path=filepath, input_shape=cfg['input_shape'], backend=backend) + model_cfg, _ = _save_model_config(model, base_path=filepath, input_shape=cfg['input_shape']) cfg['model'] = model_cfg # Serialize dataset @@ -232,7 +233,6 @@ def write_config(cfg: dict, filepath: Union[str, os.PathLike]): def _save_preprocess_config(preprocess_fn: Callable, - backend: str, input_shape: Optional[tuple], filepath: Path) -> dict: """ @@ -243,8 +243,6 @@ def _save_preprocess_config(preprocess_fn: Callable, ---------- preprocess_fn The preprocess function to be serialized. - backend - Specifies the detectors backend (if it has one). Either `'tensorflow'`, `'pytorch'` or `None`. input_shape Input shape for a model (if a model exists). filepath @@ -267,7 +265,7 @@ def _save_preprocess_config(preprocess_fn: Callable, for k, v in func_kwargs.items(): # Model/embedding if isinstance(v, SupportedModels): - cfg_model, cfg_embed = _save_model_config(v, filepath, input_shape, backend, local_path) + cfg_model, cfg_embed = _save_model_config(v, filepath, input_shape, local_path) kwargs.update({k: cfg_model}) if cfg_embed is not None: kwargs.update({'embedding': cfg_embed}) @@ -390,10 +388,9 @@ def _int2str_keys(dikt: dict) -> dict: return dikt_copy -def _save_model_config(model: Callable, +def _save_model_config(model: Any, base_path: Path, - input_shape: tuple, - backend: str, + input_shape: Optional[tuple] = None, path: Path = Path('.')) -> Tuple[dict, Optional[dict]]: """ Save a model to a config dictionary. When a model has a text embedding model contained within it, @@ -407,8 +404,6 @@ def _save_model_config(model: Callable, Base filepath to save to. input_shape The input dimensions of the model (after the optional embedding has been applied). - backend - The backend. path A local (relative) filepath to append to base_path. @@ -416,10 +411,12 @@ def _save_model_config(model: Callable, ------- A tuple containing the model and embedding config dicts. """ - if backend == 'tensorflow': + if isinstance(model, SupportedModels_tf): return save_model_config_tf(model, base_path, input_shape, path) + elif isinstance(model, SupportedModels_sklearn): + return save_model_config_sk(model, base_path, path), None else: - raise NotImplementedError("Saving of pytorch models is not yet implemented.") + raise NotImplementedError("Support for saving the given model is not yet implemented") def _save_tokenizer_config(tokenizer: PreTrainedTokenizerBase, diff --git a/alibi_detect/saving/schemas.py b/alibi_detect/saving/schemas.py index e151c0bce..722ab3d3a 100644 --- a/alibi_detect/saving/schemas.py +++ b/alibi_detect/saving/schemas.py @@ -16,10 +16,9 @@ from typing import Callable, Dict, List, Optional, Type, Union, Any -# TODO - conditional checks depending on backend etc -# TODO - consider validating output of get_config calls import numpy as np from pydantic import BaseModel, validator +from sklearn.base import BaseEstimator # import here since sklearn currently a core dep from alibi_detect.cd.tensorflow import UAE as UAE_tf from alibi_detect.cd.tensorflow import HiddenOutput as HiddenOutput_tf @@ -34,13 +33,11 @@ if has_pytorch: # import torch SupportedModels_torch = () # type: ignore # TODO - fill - # import sklearn -# SupportedModels_sklearn = () # type: ignore # TODO - fill +SupportedModels_sklearn = (BaseEstimator, ) # type: ignore # Build SupportedModels - a tuple of all possible models for use in isinstance() etc. SupportedModels = SupportedModels_tf + SupportedModels_torch + SupportedModels_sklearn -# TODO - could define a Union with fwdrefs here, for use in mypy type annotations in saving.py etc # Custom validators (defined here for reuse in multiple pydantic models) @@ -52,19 +49,33 @@ def coerce_int2list(value: int) -> List[int]: return value -def validate_model(model: Callable, values: dict) -> Callable: - """Validator to check the model is compatible with the given backend""" - backend = values['backend'] - if backend == 'tensorflow' and not isinstance(model, SupportedModels_tf): - raise ValueError('A TensorFlow backend is not available for this model') - elif backend == 'pytorch' and not isinstance(model, SupportedModels_torch): - raise ValueError('A PyTorch backend is not available for this model') - elif backend == 'sklearn' and not isinstance(model, SupportedModels_sklearn): - raise ValueError('A sklearn backend is not available for this model') - return model +class SupportedModelsType: + """ + Pydantic custom type to check the model is one of the supported types (conditional on what optional deps + are installed). + """ + @classmethod + def __get_validators__(cls): + yield cls.validate_model + + @classmethod + def validate_model(cls, model: Any, values: dict) -> Any: + backend = values['backend'] + if backend == 'tensorflow' and not isinstance(model, SupportedModels_tf): + raise TypeError("`backend='tensorflow'` but the `model` doesn't appear to be a TensorFlow supported model.") + elif backend == 'pytorch' and not isinstance(model, SupportedModels_torch): + raise TypeError("`backend='pytorch'` but the `model` doesn't appear to be a TensorFlow supported model.") + elif backend == 'sklearn' and not isinstance(model, SupportedModels_sklearn): + raise TypeError("`backend='sklearn'` but the `model` doesn't appear to be a scikit-learn supported model.") + elif isinstance(model, SupportedModels): # If model supported and no `backend` incompatibility + return model + else: # Catch any other unexpected issues + raise TypeError('The model is not recognised as a supported type.') + +# TODO - We could add validator to check `model` and `embedding` type when chained together. Leave this until refactor +# of preprocess_drift. -# TODO - we could add another validator to check given "backend" against what optional deps are installed? # Custom BaseModel so that we can set default config class CustomBaseModel(BaseModel): @@ -97,8 +108,6 @@ class DetectorConfig(CustomBaseModel): """ name: str "Name of the detector e.g. `MMDDrift`." - backend: Literal['tensorflow', 'pytorch', 'sklearn', 'keops'] = 'tensorflow' - "The detector backend." meta: Optional[MetaData] = None "Config metadata. Should not be edited." # Note: Although not all detectors have a backend, we define in base class as `backend` also determines @@ -117,9 +126,15 @@ class ModelConfig(CustomBaseModel): .. code-block :: toml [model] + flavour = "tensorflow" src = "model/" layer = -1 """ + flavour: Literal['tensorflow', 'pytorch', 'sklearn'] + """ + Whether the model is a `tensorflow`, `pytorch` or `sklearn` model. XGBoost models following the scikit-learn API + are also included under `sklearn`. + """ src: str """ Filepath to directory storing the model (relative to the `config.toml` file, or absolute). At present, @@ -135,7 +150,9 @@ class ModelConfig(CustomBaseModel): layer: Optional[int] = None """ Optional index of hidden layer to extract. If not `None`, a - :class:`~alibi_detect.cd.tensorflow.preprocess.HiddenOutput` model is returned. + :class:`~alibi_detect.cd.tensorflow.preprocess.HiddenOutput` or + :class:`~alibi_detect.cd.pytorch.preprocess.HiddenOutput` model is returned (dependent on `flavour`). + Only applies to 'tensorflow' and 'pytorch' models. """ @@ -146,16 +163,21 @@ class EmbeddingConfig(CustomBaseModel): Examples -------- - Using the hidden states at the output of each layer of the + Using the hidden states at the output of each layer of a TensorFlow `BERT base `_ model as text embeddings: .. code-block :: toml [embedding] + flavour = "tensorflow" src = "bert-base-cased" type = "hidden_state" layers = [-1, -2, -3, -4, -5, -6, -7, -8] """ + flavour: Literal['tensorflow', 'pytorch'] = 'tensorflow' + """ + Whether the embedding model is a `tensorflow` or `pytorch` model. + """ type: Literal['pooler_output', 'last_hidden_state', 'hidden_state', 'hidden_state_cls'] """ The type of embedding to be loaded. See `embedding_type` in @@ -626,6 +648,7 @@ class MMDDriftConfig(DriftDetectorConfig): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.MMDDrift` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch', 'keops'] = 'tensorflow' p_val: float = .05 preprocess_at_init: bool = True update_x_ref: Optional[Dict[str, int]] = None @@ -645,6 +668,7 @@ class MMDDriftConfigResolved(DriftDetectorConfigResolved): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.MMDDrift` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch', 'keops'] = 'tensorflow' p_val: float = .05 preprocess_at_init: bool = True update_x_ref: Optional[Dict[str, int]] = None @@ -664,6 +688,7 @@ class LSDDDriftConfig(DriftDetectorConfig): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.LSDDDrift` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch'] = 'tensorflow' p_val: float = .05 preprocess_at_init: bool = True update_x_ref: Optional[Dict[str, int]] = None @@ -682,6 +707,7 @@ class LSDDDriftConfigResolved(DriftDetectorConfigResolved): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.LSDDDrift` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch'] = 'tensorflow' p_val: float = .05 preprocess_at_init: bool = True update_x_ref: Optional[Dict[str, int]] = None @@ -701,6 +727,7 @@ class ClassifierDriftConfig(DriftDetectorConfig): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.ClassifierDrift` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch', 'sklearn'] = 'tensorflow' p_val: float = .05 preprocess_at_init: bool = True update_x_ref: Optional[Dict[str, int]] = None @@ -736,10 +763,11 @@ class ClassifierDriftConfigResolved(DriftDetectorConfigResolved): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.ClassifierDrift` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch', 'sklearn'] = 'tensorflow' p_val: float = .05 preprocess_at_init: bool = True update_x_ref: Optional[Dict[str, int]] = None - model: Optional[Callable] = None + model: Optional[SupportedModelsType] = None preds_type: Literal['probs', 'logits'] = 'probs' binarize_preds: bool = False reg_loss_fn: Optional[Callable] = None @@ -761,9 +789,6 @@ class ClassifierDriftConfigResolved(DriftDetectorConfigResolved): calibration_kwargs: Optional[dict] = None use_oob: bool = False - # validators - _validate_model = validator('model', allow_reuse=True, pre=True)(validate_model) - class SpotTheDiffDriftConfig(DriftDetectorConfig): """ @@ -774,6 +799,7 @@ class SpotTheDiffDriftConfig(DriftDetectorConfig): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.SpotTheDiffDrift` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch'] = 'tensorflow' p_val: float = .05 binarize_preds: bool = False train_size: Optional[float] = .75 @@ -805,6 +831,7 @@ class SpotTheDiffDriftConfigResolved(DriftDetectorConfigResolved): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.SpotTheDiffDrift` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch'] = 'tensorflow' p_val: float = .05 binarize_preds: bool = False train_size: Optional[float] = .75 @@ -836,6 +863,7 @@ class LearnedKernelDriftConfig(DriftDetectorConfig): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.LearnedKernelDrift` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch', 'keops'] = 'tensorflow' p_val: float = .05 kernel: Union[str, DeepKernelConfig] preprocess_at_init: bool = True @@ -868,6 +896,7 @@ class LearnedKernelDriftConfigResolved(DriftDetectorConfigResolved): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.LearnedKernelDrift` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch', 'keops'] = 'tensorflow' p_val: float = .05 kernel: Optional[Callable] = None preprocess_at_init: bool = True @@ -900,6 +929,7 @@ class ContextMMDDriftConfig(DriftDetectorConfig): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.ContextMMDDrift` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch'] = 'tensorflow' p_val: float = .05 c_ref: str preprocess_at_init: bool = True @@ -922,6 +952,7 @@ class ContextMMDDriftConfigResolved(DriftDetectorConfigResolved): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.MMDDrift` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch'] = 'tensorflow' p_val: float = .05 c_ref: np.ndarray preprocess_at_init: bool = True @@ -945,6 +976,7 @@ class MMDDriftOnlineConfig(DriftDetectorConfig): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.MMDDriftOnline` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch'] = 'tensorflow' ert: float window_size: int kernel: Optional[Union[str, KernelConfig]] = None @@ -963,6 +995,7 @@ class MMDDriftOnlineConfigResolved(DriftDetectorConfigResolved): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.MMDDriftOnline` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch'] = 'tensorflow' ert: float window_size: int kernel: Optional[Callable] = None @@ -981,6 +1014,7 @@ class LSDDDriftOnlineConfig(DriftDetectorConfig): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.LSDDDriftOnline` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch'] = 'tensorflow' ert: float window_size: int sigma: Optional[np.ndarray] = None @@ -1000,6 +1034,7 @@ class LSDDDriftOnlineConfigResolved(DriftDetectorConfigResolved): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.LSDDDriftOnline` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch'] = 'tensorflow' ert: float window_size: int sigma: Optional[np.ndarray] = None @@ -1105,6 +1140,7 @@ class ClassifierUncertaintyDriftConfig(DetectorConfig): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.ClassifierUncertaintyDrift` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch'] = 'tensorflow' x_ref: str model: Union[str, ModelConfig] p_val: float = .05 @@ -1131,8 +1167,9 @@ class ClassifierUncertaintyDriftConfigResolved(DetectorConfig): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.ClassifierUncertaintyDrift` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch'] = 'tensorflow' x_ref: Union[np.ndarray, list] - model: Optional[Callable] = None + model: Optional[SupportedModelsType] = None p_val: float = .05 x_ref_preprocessed: bool = False update_x_ref: Optional[Dict[str, int]] = None @@ -1147,9 +1184,6 @@ class ClassifierUncertaintyDriftConfigResolved(DetectorConfig): input_shape: Optional[tuple] = None data_type: Optional[str] = None - # validators - _validate_model = validator('model', allow_reuse=True, pre=True)(validate_model) - class RegressorUncertaintyDriftConfig(DetectorConfig): """ @@ -1160,6 +1194,7 @@ class RegressorUncertaintyDriftConfig(DetectorConfig): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.RegressorUncertaintyDrift` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch'] = 'tensorflow' x_ref: str model: Union[str, ModelConfig] p_val: float = .05 @@ -1185,8 +1220,9 @@ class RegressorUncertaintyDriftConfigResolved(DetectorConfig): Except for the `name` and `meta` fields, the fields match the detector's args and kwargs. Refer to the :class:`~alibi_detect.cd.RegressorUncertaintyDrift` documentation for a description of each field. """ + backend: Literal['tensorflow', 'pytorch'] = 'tensorflow' x_ref: Union[np.ndarray, list] - model: Optional[Callable] = None + model: Optional[SupportedModelsType] = None p_val: float = .05 x_ref_preprocessed: bool = False update_x_ref: Optional[Dict[str, int]] = None @@ -1200,9 +1236,6 @@ class RegressorUncertaintyDriftConfigResolved(DetectorConfig): input_shape: Optional[tuple] = None data_type: Optional[str] = None - # validators - _validate_model = validator('model', allow_reuse=True, pre=True)(validate_model) - # Unresolved schema dictionary (used in alibi_detect.utils.loading) DETECTOR_CONFIGS = { diff --git a/alibi_detect/saving/tests/models.py b/alibi_detect/saving/tests/models.py new file mode 100644 index 000000000..9857449ad --- /dev/null +++ b/alibi_detect/saving/tests/models.py @@ -0,0 +1,251 @@ +from functools import partial + +import numpy as np +import tensorflow as tf +import torch +from sklearn.ensemble import RandomForestClassifier +from xgboost import XGBClassifier + +from requests.exceptions import HTTPError + +import pytest +from pytest_cases import fixture, parametrize +from transformers import AutoTokenizer +from alibi_detect.cd.pytorch import preprocess_drift as preprocess_drift_pt +from alibi_detect.cd.tensorflow import UAE as UAE_tf +from alibi_detect.cd.tensorflow import preprocess_drift as preprocess_drift_tf +from alibi_detect.utils.pytorch.kernels import GaussianRBF as GaussianRBF_pt +from alibi_detect.utils.tensorflow.kernels import GaussianRBF as GaussianRBF_tf +from alibi_detect.utils.tensorflow.kernels import DeepKernel as DeepKernel_tf +from alibi_detect.models.pytorch import TransformerEmbedding as TransformerEmbedding_pt +from alibi_detect.models.tensorflow import TransformerEmbedding as TransformerEmbedding_tf +from alibi_detect.cd.pytorch import HiddenOutput as HiddenOutput_pt +from alibi_detect.cd.tensorflow import HiddenOutput as HiddenOutput_tf + +LATENT_DIM = 2 # Must be less than input_dim set in ./datasets.py +DEVICE = "cuda" if torch.cuda.is_available() else "cpu" + + +@fixture +def encoder_model(backend, current_cases): + """ + An untrained encoder of given input dimension and backend (this is a "custom" model, NOT an Alibi Detect UAE). + """ + _, _, data_params = current_cases["data"] + _, input_dim = data_params['data_shape'] + + if backend == 'tensorflow': + model = tf.keras.Sequential( + [ + tf.keras.layers.InputLayer(input_shape=(input_dim,)), + tf.keras.layers.Dense(5, activation=tf.nn.relu), + tf.keras.layers.Dense(LATENT_DIM, activation=None) + ] + ) + elif backend == 'pytorch': + raise NotImplementedError('`pytorch` tests not implemented.') + else: + pytest.skip('`encoder_model` only implemented for tensorflow and pytorch.') + return model + + +@fixture +def encoder_dropout_model(backend, current_cases): + """ + An untrained encoder with dropout, of given input dimension and backend. + + TODO: consolidate this model (and encoder_model above) with models like that in test_model_uncertainty.py + """ + _, _, data_params = current_cases["data"] + _, input_dim = data_params['data_shape'] + + if backend == 'tensorflow': + model = tf.keras.Sequential( + [ + tf.keras.layers.InputLayer(input_shape=(input_dim,)), + tf.keras.layers.Dense(5, activation=tf.nn.relu), + tf.keras.layers.Dropout(0.5), + tf.keras.layers.Dense(LATENT_DIM, activation=None) + ] + ) + elif backend == 'pytorch': + raise NotImplementedError('`pytorch` tests not implemented.') + else: + pytest.skip('`encoder_dropout_model` only implemented for tensorflow and pytorch.') + return model + + +@fixture +def preprocess_custom(encoder_model): + """ + Preprocess function with Untrained Autoencoder. + """ + if isinstance(encoder_model, tf.keras.Model): + preprocess_fn = partial(preprocess_drift_tf, model=encoder_model) + else: + preprocess_fn = partial(preprocess_drift_pt, model=encoder_model) + return preprocess_fn + + +@fixture +def kernel(request, backend): + """ + Gaussian RBF kernel for given backend. Settings are parametrised in the test function. + """ + kernel = request.param + if kernel is None: + pass + elif isinstance(kernel, dict): # dict of kwargs + if backend == 'tensorflow': + kernel = GaussianRBF_tf(**kernel) + elif backend == 'pytorch': + kernel = GaussianRBF_pt(**kernel) + else: + pytest.skip('`kernel` only implemented for tensorflow and pytorch.') + return kernel + + +@fixture +def deep_kernel(request, backend, encoder_model): + """ + Deep kernel, built using the `encoder_model` fixture for the projection, and using the kernel_a and eps + parametrised in the test function. + """ + # Get DeepKernel options + kernel_a = request.param.get('kernel_a', 'rbf') + kernel_b = request.param.get('kernel_b', 'rbf') + eps = request.param.get('eps', 'trainable') + + # Proj model (backend managed in encoder_model fixture) + proj = encoder_model + + # Build DeepKernel + if backend == 'tensorflow': + kernel_a = GaussianRBF_tf(**kernel_a) if isinstance(kernel_a, dict) else kernel_a + kernel_a = GaussianRBF_tf(**kernel_b) if isinstance(kernel_b, dict) else kernel_b + deep_kernel = DeepKernel_tf(proj, kernel_a=kernel_a, kernel_b=kernel_b, eps=eps) + elif backend == 'pytorch': + raise NotImplementedError('`pytorch` tests not implemented.') + else: + pytest.skip('`deep_kernel` only implemented for tensorflow and pytorch.') + return deep_kernel + + +@fixture +def classifier_model(backend, current_cases): + """ + Classification model with given input dimension and backend. + """ + _, _, data_params = current_cases["data"] + _, input_dim = data_params['data_shape'] + if backend == 'tensorflow': + inputs = tf.keras.Input(shape=(input_dim,)) + outputs = tf.keras.layers.Dense(2, activation=tf.nn.softmax)(inputs) + model = tf.keras.Model(inputs=inputs, outputs=outputs) + elif backend == 'pytorch': + raise NotImplementedError('`pytorch` tests not implemented.') + elif backend == 'sklearn': + model = RandomForestClassifier() + else: + pytest.skip('`classifier_model` only implemented for tensorflow, pytorch, and sklearn.') + return model + + +@fixture +def xgb_classifier_model(): + model = XGBClassifier() + return model + + +@fixture(unpack_into=('tokenizer, embedding, max_len, enc_dim')) +@parametrize('model_name, max_len', [('bert-base-cased', 100)]) +@parametrize('uae', [True, False]) +def nlp_embedding_and_tokenizer(model_name, max_len, uae, backend): + """ + A fixture to build nlp embedding and tokenizer models based on the HuggingFace pre-trained models. + """ + backend = 'tf' if backend == 'tensorflow' else 'pt' + + # Load tokenizer + try: + tokenizer = AutoTokenizer.from_pretrained(model_name) + except (OSError, HTTPError): + pytest.skip(f"Problem downloading {model_name} from huggingface.co") + X = 'A dummy string' # this will be padded to max_len + tokens = tokenizer(list(X[:5]), pad_to_max_length=True, + max_length=max_len, return_tensors=backend) + + # Load embedding model + emb_type = 'hidden_state' + n_layers = 8 + layers = [-_ for _ in range(1, n_layers + 1)] + enc_dim = 32 + + if backend == 'tf': + try: + embedding = TransformerEmbedding_tf(model_name, emb_type, layers) + except (OSError, HTTPError): + pytest.skip(f"Problem downloading {model_name} from huggingface.co") + if uae: + x_emb = embedding(tokens) + shape = (x_emb.shape[1],) + embedding = UAE_tf(input_layer=embedding, shape=shape, enc_dim=enc_dim) + else: + try: + embedding = TransformerEmbedding_pt(model_name, emb_type, layers) + except (OSError, HTTPError): + pytest.skip(f"Problem downloading {model_name} from huggingface.co") + if uae: + x_emb = embedding(tokens) + emb_dim = x_emb.shape[1] + device = torch.device(DEVICE) + embedding = torch.nn.Sequential( + embedding, + torch.nn.Linear(emb_dim, 256), + torch.nn.ReLU(), + torch.nn.Linear(256, enc_dim) + ).to(device).eval() + + return tokenizer, embedding, max_len, enc_dim + + +def preprocess_simple(x: np.ndarray): + """ + Simple function to test serialization of generic Python function within preprocess_fn. + """ + return x*2.0 + + +@fixture +def preprocess_nlp(embedding, tokenizer, max_len, backend): + """ + Preprocess function with Untrained Autoencoder. + """ + if backend == 'tensorflow': + preprocess_fn = partial(preprocess_drift_tf, model=embedding, tokenizer=tokenizer, + max_len=max_len, preprocess_batch_fn=preprocess_simple) + elif backend == 'pytorch': + preprocess_fn = partial(preprocess_drift_pt, model=embedding, tokenizer=tokenizer, max_len=max_len, + preprocess_batch_fn=preprocess_simple) + else: + pytest.skip('`preprocess_nlp` only implemented for tensorflow and pytorch.') + return preprocess_fn + + +@fixture +def preprocess_hiddenoutput(classifier_model, current_cases, backend): + """ + Preprocess function to extract the softmax layer of a classifier (with the HiddenOutput utility function). + """ + _, _, data_params = current_cases["data"] + _, input_dim = data_params['data_shape'] + + if backend == 'tensorflow': + model = HiddenOutput_tf(classifier_model, layer=-1, input_shape=(None, input_dim)) + preprocess_fn = partial(preprocess_drift_tf, model=model) + elif backend == 'pytorch': + model = HiddenOutput_pt(classifier_model, layer=-1) + preprocess_fn = partial(preprocess_drift_pt, model=model) + else: + pytest.skip('`preprocess_hiddenoutput` only implemented for tensorflow and pytorch.') + return preprocess_fn diff --git a/alibi_detect/saving/tests/test_saving.py b/alibi_detect/saving/tests/test_saving.py index 3d08cfd96..c1b6b5a06 100644 --- a/alibi_detect/saving/tests/test_saving.py +++ b/alibi_detect/saving/tests/test_saving.py @@ -5,12 +5,11 @@ Internal functions such as save_kernel/load_kernel_config etc are also tested. """ # TODO future - test pytorch save/load functionality -# TODO (could/should also add tests to backend-specific submodules) from functools import partial from pathlib import Path from typing import Callable -from requests.exceptions import HTTPError +import sklearn.base import toml import dill import numpy as np @@ -20,47 +19,44 @@ import torch from .datasets import BinData, CategoricalData, ContinuousData, MixedData, TextData +from .models import (encoder_model, preprocess_custom, preprocess_hiddenoutput, preprocess_simple, # noqa: F401 + preprocess_nlp, LATENT_DIM, classifier_model, kernel, deep_kernel, nlp_embedding_and_tokenizer, + embedding, tokenizer, max_len, enc_dim) + from alibi_detect.utils._random import fixed_seed from packaging import version -from pytest_cases import fixture, param_fixture, parametrize, parametrize_with_cases +from pytest_cases import param_fixture, parametrize, parametrize_with_cases from sklearn.model_selection import StratifiedKFold -from transformers import AutoTokenizer + from alibi_detect.cd import (ChiSquareDrift, ClassifierUncertaintyDrift, RegressorUncertaintyDrift, ClassifierDrift, FETDrift, KSDrift, LearnedKernelDrift, LSDDDrift, MMDDrift, SpotTheDiffDrift, TabularDrift, ContextMMDDrift, MMDDriftOnline, LSDDDriftOnline, CVMDriftOnline, FETDriftOnline) -from alibi_detect.cd.pytorch import HiddenOutput as HiddenOutput_pt -from alibi_detect.cd.pytorch import preprocess_drift as preprocess_drift_pt -from alibi_detect.cd.tensorflow import UAE as UAE_tf -from alibi_detect.cd.tensorflow import HiddenOutput as HiddenOutput_tf -from alibi_detect.cd.tensorflow import preprocess_drift as preprocess_drift_tf from alibi_detect.models.pytorch import TransformerEmbedding as TransformerEmbedding_pt from alibi_detect.models.tensorflow import TransformerEmbedding as TransformerEmbedding_tf from alibi_detect.saving import (load_detector, read_config, registry, resolve_config, save_detector, write_config) -from alibi_detect.saving.loading import (_get_nested_value, _load_model_config, _load_optimizer_config, _replace, +from alibi_detect.saving.loading import (_get_nested_value, _replace, _set_dtypes, _set_nested_value, _prepend_cfg_filepaths) from alibi_detect.saving.saving import _serialize_object from alibi_detect.saving.saving import (_path2str, _int2str_keys, _save_kernel_config, _save_model_config, _save_preprocess_config) from alibi_detect.saving.schemas import DeepKernelConfig, KernelConfig, ModelConfig, PreprocessConfig from alibi_detect.utils.pytorch.kernels import DeepKernel as DeepKernel_pt -from alibi_detect.utils.pytorch.kernels import GaussianRBF as GaussianRBF_pt from alibi_detect.utils.tensorflow.kernels import DeepKernel as DeepKernel_tf -from alibi_detect.utils.tensorflow.kernels import GaussianRBF as GaussianRBF_tf if version.parse(scipy.__version__) >= version.parse('1.7.0'): from alibi_detect.cd import CVMDrift -backend = param_fixture("backend", ['tensorflow']) +# TODO: We currently parametrize encoder_model etc (in models.py) with backend, so the same flavour of +# preprocessing is used as the detector backend. In the future we could decouple this in tests. +backend = param_fixture("backend", ['tensorflow', 'sklearn']) P_VAL = 0.05 ERT = 10 N_PERMUTATIONS = 10 N_BOOTSTRAPS = 100 WINDOW_SIZE = 5 -LATENT_DIM = 2 # Must be less than input_dim set in ./datasets.py -DEVICE = "cuda" if torch.cuda.is_available() else "cpu" REGISTERED_OBJECTS = registry.get_all() # Define a detector config dict @@ -76,213 +72,6 @@ # TODO - future: Some of the fixtures can/should be moved elsewhere (i.e. if they can be recycled for use elsewhere) -@fixture -def encoder_model(backend, current_cases): - """ - An untrained encoder of given input dimension and backend (this is a "custom" model, NOT an Alibi Detect UAE). - """ - _, _, data_params = current_cases["data"] - _, input_dim = data_params['data_shape'] - - if backend == 'tensorflow': - model = tf.keras.Sequential( - [ - tf.keras.layers.InputLayer(input_shape=(input_dim,)), - tf.keras.layers.Dense(5, activation=tf.nn.relu), - tf.keras.layers.Dense(LATENT_DIM, activation=None) - ] - ) - else: - raise NotImplementedError('`pytorch` tests not implemented.') - return model - - -@fixture -def encoder_dropout_model(backend, current_cases): - """ - An untrained encoder with dropout, of given input dimension and backend. - - TODO: consolidate this model (and encoder_model above) with models like that in test_model_uncertainty.py - """ - _, _, data_params = current_cases["data"] - _, input_dim = data_params['data_shape'] - - if backend == 'tensorflow': - model = tf.keras.Sequential( - [ - tf.keras.layers.InputLayer(input_shape=(input_dim,)), - tf.keras.layers.Dense(5, activation=tf.nn.relu), - tf.keras.layers.Dropout(0.5), - tf.keras.layers.Dense(LATENT_DIM, activation=None) - ] - ) - else: - raise NotImplementedError('`pytorch` tests not implemented.') - return model - - -@fixture -def preprocess_custom(encoder_model, backend): - """ - Preprocess function with Untrained Autoencoder. - """ - if backend == 'tensorflow': - preprocess_fn = partial(preprocess_drift_tf, model=encoder_model) - else: - preprocess_fn = partial(preprocess_drift_pt, model=encoder_model) - return preprocess_fn - - -@fixture -def kernel(request, backend): - """ - Gaussian RBF kernel for given backend. Settings are parametrised in the test function. - """ - kernel = request.param - if kernel is None: - pass - elif isinstance(kernel, dict): # dict of kwargs - if backend == 'tensorflow': - kernel = GaussianRBF_tf(**kernel) - elif backend == 'pytorch': - kernel = GaussianRBF_pt(**kernel) - return kernel - - -@fixture -def deep_kernel(request, backend, encoder_model): - """ - Deep kernel, built using the `encoder_model` fixture for the projection, and using the kernel_a and eps - parametrised in the test function. - """ - # Get DeepKernel options - kernel_a = request.param.get('kernel_a', 'rbf') - kernel_b = request.param.get('kernel_b', 'rbf') - eps = request.param.get('eps', 'trainable') - - # Proj model (backend managed in encoder_model fixture) - proj = encoder_model - - # Build DeepKernel - if backend == 'tensorflow': - kernel_a = GaussianRBF_tf(**kernel_a) if isinstance(kernel_a, dict) else kernel_a - kernel_a = GaussianRBF_tf(**kernel_b) if isinstance(kernel_b, dict) else kernel_b - deep_kernel = DeepKernel_tf(proj, kernel_a=kernel_a, kernel_b=kernel_b, eps=eps) - elif backend == 'pytorch': - raise NotImplementedError('`pytorch` tests not implemented.') - else: - raise ValueError('`backend` not valid.') - return deep_kernel - - -@fixture -def classifier_model(backend, current_cases): - """ - Classification model with given input dimension and backend. - """ - _, _, data_params = current_cases["data"] - _, input_dim = data_params['data_shape'] - if backend == 'tensorflow': - inputs = tf.keras.Input(shape=(input_dim,)) - outputs = tf.keras.layers.Dense(2, activation=tf.nn.softmax)(inputs) - model = tf.keras.Model(inputs=inputs, outputs=outputs) - elif backend == 'pytorch': - raise NotImplementedError('`pytorch` tests not implemented.') - else: - raise ValueError('`backend` not valid.') - return model - - -@fixture(unpack_into=('tokenizer, embedding, max_len, enc_dim')) -@parametrize('model_name, max_len', [('bert-base-cased', 100)]) -@parametrize('uae', [True, False]) -def nlp_embedding_and_tokenizer(model_name, max_len, uae, backend): - """ - A fixture to build nlp embedding and tokenizer models based on the HuggingFace pre-trained models. - """ - backend = 'tf' if backend == 'tensorflow' else 'pt' - - # Load tokenizer - try: - tokenizer = AutoTokenizer.from_pretrained(model_name) - except (OSError, HTTPError): - pytest.skip(f"Problem downloading {model_name} from huggingface.co") - X = 'A dummy string' # this will be padded to max_len - tokens = tokenizer(list(X[:5]), pad_to_max_length=True, - max_length=max_len, return_tensors=backend) - - # Load embedding model - emb_type = 'hidden_state' - n_layers = 8 - layers = [-_ for _ in range(1, n_layers + 1)] - enc_dim = 32 - - if backend == 'tf': - try: - embedding = TransformerEmbedding_tf(model_name, emb_type, layers) - except (OSError, HTTPError): - pytest.skip(f"Problem downloading {model_name} from huggingface.co") - if uae: - x_emb = embedding(tokens) - shape = (x_emb.shape[1],) - embedding = UAE_tf(input_layer=embedding, shape=shape, enc_dim=enc_dim) - else: - try: - embedding = TransformerEmbedding_pt(model_name, emb_type, layers) - except (OSError, HTTPError): - pytest.skip(f"Problem downloading {model_name} from huggingface.co") - if uae: - x_emb = embedding(tokens) - emb_dim = x_emb.shape[1] - device = torch.device(DEVICE) - embedding = torch.nn.Sequential( - embedding, - torch.nn.Linear(emb_dim, 256), - torch.nn.ReLU(), - torch.nn.Linear(256, enc_dim) - ).to(device).eval() - - return tokenizer, embedding, max_len, enc_dim - - -def preprocess_simple(x: np.ndarray): - """ - Simple function to test serialization of generic Python function within preprocess_fn. - """ - return x*2.0 - - -@fixture -def preprocess_nlp(embedding, tokenizer, max_len, backend): - """ - Preprocess function with Untrained Autoencoder. - """ - if backend == 'tensorflow': - preprocess_fn = partial(preprocess_drift_tf, model=embedding, tokenizer=tokenizer, - max_len=max_len, preprocess_batch_fn=preprocess_simple) - else: - preprocess_fn = partial(preprocess_drift_pt, model=embedding, tokenizer=tokenizer, max_len=max_len, - preprocess_batch_fn=preprocess_simple) - return preprocess_fn - - -@fixture -def preprocess_hiddenoutput(classifier_model, current_cases, backend): - """ - Preprocess function to extract the softmax layer of a classifier (with the HiddenOutput utility function). - """ - _, _, data_params = current_cases["data"] - _, input_dim = data_params['data_shape'] - - if backend == 'tensorflow': - model = HiddenOutput_tf(classifier_model, layer=-1, input_shape=(None, input_dim)) - preprocess_fn = partial(preprocess_drift_tf, model=model) - else: - model = HiddenOutput_pt(classifier_model, layer=-1) - preprocess_fn = partial(preprocess_drift_pt, model=model) - return preprocess_fn - - @parametrize('cfg', CFGS) def test_load_simple_config(cfg, tmp_path): """ @@ -338,9 +127,10 @@ def test_save_ksdrift(data, preprocess_fn, tmp_path): cd_load.predict(X_h0)['data']['p_val']) +@pytest.mark.skipif(backend == 'sklearn', reason="Don't test with sklearn preprocessing.") @parametrize('preprocess_fn', [preprocess_nlp]) @parametrize_with_cases("data", cases=TextData.movie_sentiment_data, prefix='data_') -def test_save_ksdrift_nlp(data, preprocess_fn, enc_dim, tmp_path): +def test_save_ksdrift_nlp(data, preprocess_fn, enc_dim, tmp_path): # noqa: F811 """ Test KSDrift on continuous datasets, with UAE and classifier_model softmax output as preprocess_fn's. Only this detector is tested with embedding and embedding+uae, as other detectors should see the same preprocessed data. @@ -406,12 +196,15 @@ def test_save_cvmdrift(data, preprocess_custom, tmp_path): ], indirect=True ) @parametrize_with_cases("data", cases=ContinuousData, prefix='data_') -def test_save_mmddrift(data, kernel, preprocess_custom, backend, tmp_path, seed): +def test_save_mmddrift(data, kernel, preprocess_custom, backend, tmp_path, seed): # noqa: F811 """ Test MMDDrift on continuous datasets, with UAE as preprocess_fn. Detector is saved and then loaded, with assertions checking that the reinstantiated detector is equivalent. """ + if backend not in ('tensorflow', 'pytorch', 'keops'): + pytest.skip("Detector doesn't have this backend") + # Init detector and make predictions X_ref, X_h0 = data with fixed_seed(seed): @@ -454,6 +247,9 @@ def test_save_lsdddrift(data, preprocess_at_init, backend, tmp_path, seed): Detector is saved and then loaded, with assertions checking that the reinstantiated detector is equivalent. """ + if backend not in ('tensorflow', 'pytorch'): + pytest.skip("Detector doesn't have this backend") + preprocess_fn = preprocess_simple # TODO - TensorFlow based preprocessors currently cause in-deterministic behaviour with LSDD permutations. Replace # preprocess_simple with parametrized preprocess_fn's once above issue resolved. @@ -575,8 +371,11 @@ def test_save_tabulardrift(data, tmp_path): @parametrize_with_cases("data", cases=ContinuousData, prefix='data_') -def test_save_classifierdrift(data, classifier_model, backend, tmp_path, seed): +def test_save_classifierdrift(data, classifier_model, backend, tmp_path, seed): # noqa: F811 """ Test ClassifierDrift on continuous datasets.""" + if backend not in ('tensorflow', 'pytorch', 'sklearn'): + pytest.skip("Detector doesn't have this backend") + # Init detector and predict X_ref, X_h0 = data with fixed_seed(seed): @@ -598,9 +397,12 @@ def test_save_classifierdrift(data, classifier_model, backend, tmp_path, seed): np.testing.assert_array_equal(X_ref, cd_load._detector.x_ref) assert isinstance(cd_load._detector.skf, StratifiedKFold) assert cd_load._detector.p_val == P_VAL - assert isinstance(cd_load._detector.train_kwargs, dict) + if backend != 'sklearn': + assert isinstance(cd_load._detector.train_kwargs, dict) if backend == 'tensorflow': assert isinstance(cd_load._detector.model, tf.keras.Model) + elif backend == 'sklearn': + assert isinstance(cd_load._detector.model, sklearn.base.BaseEstimator) else: pass # TODO # TODO - detector still not deterministic, investigate in future @@ -609,12 +411,15 @@ def test_save_classifierdrift(data, classifier_model, backend, tmp_path, seed): @parametrize_with_cases("data", cases=ContinuousData, prefix='data_') -def test_save_spotthediff(data, classifier_model, backend, tmp_path, seed): +def test_save_spotthediff(data, classifier_model, backend, tmp_path, seed): # noqa: F811 """ Test SpotTheDiffDrift on continuous datasets. Detector is saved and then loaded, with assertions checking that the reinstantiated detector is equivalent. """ + if backend not in ('tensorflow', 'pytorch'): + pytest.skip("Detector doesn't have this backend") + # Init detector and predict X_ref, X_h0 = data with fixed_seed(seed): @@ -650,12 +455,15 @@ def test_save_spotthediff(data, classifier_model, backend, tmp_path, seed): ], indirect=True ) @parametrize_with_cases("data", cases=ContinuousData, prefix='data_') -def test_save_learnedkernel(data, deep_kernel, backend, tmp_path, seed): +def test_save_learnedkernel(data, deep_kernel, backend, tmp_path, seed): # noqa: F811 """ Test LearnedKernelDrift on continuous datasets. Detector is saved and then loaded, with assertions checking that the reinstantiated detector is equivalent. """ + if backend not in ('tensorflow', 'pytorch', 'keops'): + pytest.skip("Detector doesn't have this backend") + # Init detector and predict X_ref, X_h0 = data with fixed_seed(seed): @@ -690,12 +498,15 @@ def test_save_learnedkernel(data, deep_kernel, backend, tmp_path, seed): ], indirect=True ) @parametrize_with_cases("data", cases=ContinuousData, prefix='data_') -def test_save_contextmmddrift(data, kernel, backend, tmp_path, seed): +def test_save_contextmmddrift(data, kernel, backend, tmp_path, seed): # noqa: F811 """ Test ContextMMDDrift on continuous datasets, with UAE as preprocess_fn. Detector is saved and then loaded, with assertions checking that the reinstantiated detector is equivalent. """ + if backend not in ('tensorflow', 'pytorch'): + pytest.skip("Detector doesn't have this backend") + # Init detector and make predictions X_ref, X_h0 = data C_ref, C_h0 = (X_ref[:, 0] + 1).reshape(-1, 1), (X_h0[:, 0] + 1).reshape(-1, 1) @@ -734,8 +545,11 @@ def test_save_contextmmddrift(data, kernel, backend, tmp_path, seed): @parametrize_with_cases("data", cases=ContinuousData, prefix='data_') -def test_save_classifieruncertaintydrift(data, classifier_model, backend, tmp_path, seed): +def test_save_classifieruncertaintydrift(data, classifier_model, backend, tmp_path, seed): # noqa: F811 """ Test ClassifierDrift on continuous datasets.""" + if backend not in ('tensorflow', 'pytorch'): + pytest.skip("Detector doesn't have this backend") + # Init detector and predict X_ref, X_h0 = data with fixed_seed(seed): @@ -764,6 +578,9 @@ def test_save_classifieruncertaintydrift(data, classifier_model, backend, tmp_pa @parametrize('regressor', [encoder_model]) def test_save_regressoruncertaintydrift(data, regressor, backend, tmp_path, seed): """ Test RegressorDrift on continuous datasets.""" + if backend not in ('tensorflow', 'pytorch'): + pytest.skip("Detector doesn't have this backend") + # Init detector and predict X_ref, X_h0 = data with fixed_seed(seed): @@ -794,12 +611,15 @@ def test_save_regressoruncertaintydrift(data, regressor, backend, tmp_path, seed ], indirect=True ) @parametrize_with_cases("data", cases=ContinuousData, prefix='data_') -def test_save_onlinemmddrift(data, kernel, preprocess_custom, backend, tmp_path, seed): +def test_save_onlinemmddrift(data, kernel, preprocess_custom, backend, tmp_path, seed): # noqa: F811 """ Test MMDDriftOnline on continuous datasets, with UAE as preprocess_fn. Detector is saved and then loaded, with assertions checking that the reinstantiated detector is equivalent. """ + if backend not in ('tensorflow', 'pytorch'): + pytest.skip("Detector doesn't have this backend") + # Init detector and make predictions X_ref, X_h0 = data @@ -846,6 +666,9 @@ def test_save_onlinelsdddrift(data, preprocess_custom, backend, tmp_path, seed): Detector is saved and then loaded, with assertions checking that the reinstantiated detector is equivalent. """ + if backend not in ('tensorflow', 'pytorch'): + pytest.skip("Detector doesn't have this backend") + # Init detector and make predictions X_ref, X_h0 = data @@ -891,6 +714,9 @@ def test_save_onlinecvmdrift(data, preprocess_custom, tmp_path, seed): Detector is saved and then loaded, with assertions checking that the reinstantiated detector is equivalent. """ + if backend not in ('tensorflow', 'pytorch'): + pytest.skip("Detector doesn't have this backend") + # Init detector and make predictions X_ref, X_h0 = data @@ -933,6 +759,9 @@ def test_save_onlinefetdrift(data, tmp_path, seed): Detector is saved and then loaded, with assertions checking that the reinstantiated detector is equivalent. """ + if backend not in ('tensorflow', 'pytorch'): + pytest.skip("Detector doesn't have this backend") + # Init detector and make predictions X_ref, X_h0 = data @@ -1023,7 +852,7 @@ def test_version_warning(data, tmp_path): {'sigma': None, 'trainable': True, 'init_sigma_fn': None}, ], indirect=True ) -def test_save_kernel(kernel, backend, tmp_path): +def test_save_kernel(kernel, backend, tmp_path): # noqa: F811 """ Unit test for _save/_load_kernel_config, when kernel is a GaussianRBF kernel. @@ -1066,7 +895,7 @@ def test_save_kernel(kernel, backend, tmp_path): {'kernel_a': {'trainable': True}, 'kernel_b': 'rbf', 'eps': 0.01}, # Explicit kernel_a, fixed eps ], indirect=True ) -def test_save_deepkernel(data, deep_kernel, backend, tmp_path): +def test_save_deepkernel(data, deep_kernel, backend, tmp_path): # noqa: F811 """ Unit test for _save/_load_kernel_config, when kernel is a DeepKernel kernel. @@ -1080,8 +909,7 @@ def test_save_deepkernel(data, deep_kernel, backend, tmp_path): filepath = tmp_path filename = 'mykernel' cfg_kernel = _save_kernel_config(deep_kernel, filepath, filename) - cfg_kernel['proj'], _ = _save_model_config(cfg_kernel['proj'], base_path=filepath, input_shape=input_shape, - backend=backend) + cfg_kernel['proj'], _ = _save_model_config(cfg_kernel['proj'], base_path=filepath, input_shape=input_shape) cfg_kernel = _path2str(cfg_kernel) cfg_kernel['proj'] = ModelConfig(**cfg_kernel['proj']).dict() # Pass thru ModelConfig to set `layers` etc cfg_kernel = DeepKernelConfig(**cfg_kernel).dict() # pydantic validation @@ -1120,10 +948,7 @@ def test_save_preprocess(data, preprocess_fn, tmp_path, backend): filepath = tmp_path X_ref, X_h0 = data input_shape = (X_ref.shape[1],) - cfg_preprocess = _save_preprocess_config(preprocess_fn, - backend=backend, - input_shape=input_shape, - filepath=filepath) + cfg_preprocess = _save_preprocess_config(preprocess_fn, input_shape=input_shape, filepath=filepath) cfg_preprocess = _path2str(cfg_preprocess) cfg_preprocess = PreprocessConfig(**cfg_preprocess).dict() # pydantic validation assert cfg_preprocess['src'] == '@cd.' + backend + '.preprocess.preprocess_drift' @@ -1150,7 +975,6 @@ def test_save_preprocess_nlp(data, preprocess_fn, tmp_path, backend): # Save preprocess_fn to config filepath = tmp_path cfg_preprocess = _save_preprocess_config(preprocess_fn, - backend=backend, input_shape=(768,), # hardcoded to bert-base-cased for now filepath=filepath) cfg_preprocess = _path2str(cfg_preprocess) @@ -1170,69 +994,14 @@ def test_save_preprocess_nlp(data, preprocess_fn, tmp_path, backend): assert isinstance(preprocess_fn_load.keywords['tokenizer'], type(preprocess_fn.keywords['tokenizer'])) assert isinstance(preprocess_fn_load.keywords['model'], type(preprocess_fn.keywords['model'])) if isinstance(preprocess_fn.keywords['model'], (TransformerEmbedding_tf, TransformerEmbedding_pt)): - embedding = preprocess_fn.keywords['model'] - embedding_load = preprocess_fn_load.keywords['model'] + emb = preprocess_fn.keywords['model'] + emb_load = preprocess_fn_load.keywords['model'] else: - embedding = preprocess_fn.keywords['model'].encoder.layers[0] - embedding_load = preprocess_fn_load.keywords['model'].encoder.layers[0] - assert isinstance(embedding_load.model, type(embedding.model)) - assert embedding_load.emb_type == embedding.emb_type - assert embedding_load.hs_emb.keywords['layers'] == embedding.hs_emb.keywords['layers'] - - -@parametrize_with_cases("data", cases=ContinuousData.data_synthetic_nd, prefix='data_') -@parametrize('model', [encoder_model]) -@parametrize('layer', [None, -1]) -def test_save_model(data, model, layer, backend, tmp_path): - """ - Unit test for _save_model_config and _load_model_config. - """ - # Save model - filepath = tmp_path - input_shape = (data[0].shape[1],) - cfg_model, _ = _save_model_config(model, base_path=filepath, input_shape=input_shape, backend=backend) - cfg_model = _path2str(cfg_model) - cfg_model = ModelConfig(**cfg_model).dict() - assert tmp_path.joinpath('model').is_dir() - assert tmp_path.joinpath('model/model.h5').is_file() - - # Adjust config - cfg_model['src'] = tmp_path.joinpath('model') # Need to manually set to absolute path here - if layer is not None: - cfg_model['layer'] = layer - - # Load model - model_load = _load_model_config(cfg_model, backend=backend) - if layer is None: - assert isinstance(model_load, type(model)) - else: - assert isinstance(model_load, (HiddenOutput_tf, HiddenOutput_pt)) - - -def test_save_optimizer(backend): - class_name = 'Adam' - learning_rate = 0.01 - epsilon = 1e-7 - amsgrad = False - - if backend == 'tensorflow': - # Load - cfg_opt = { - 'class_name': class_name, - 'config': { - 'name': class_name, - 'learning_rate': learning_rate, - 'epsilon': epsilon, - 'amsgrad': amsgrad - } - } - optimizer = _load_optimizer_config(cfg_opt, backend=backend) - assert type(optimizer).__name__ == class_name - assert optimizer.learning_rate == learning_rate - assert optimizer.epsilon == epsilon - assert optimizer.amsgrad == amsgrad - - # TODO - pytorch + emb = preprocess_fn.keywords['model'].encoder.layers[0] + emb_load = preprocess_fn_load.keywords['model'].encoder.layers[0] + assert isinstance(emb_load.model, type(emb.model)) + assert emb_load.emb_type == emb.emb_type + assert emb_load.hs_emb.keywords['layers'] == emb.hs_emb.keywords['layers'] def test_nested_value(): @@ -1361,6 +1130,9 @@ def test_set_dtypes(backend): dtype = 'tf.float32' elif backend == 'pytorch': dtype = 'torch.float32' + else: + pytest.skip('Only test set_dtypes for tensorflow and pytorch.') + cfg = { 'preprocess_fn': { 'dtype': dtype diff --git a/doc/source/overview/saving.md b/doc/source/overview/saving.md index 19078652a..38095e0c0 100644 --- a/doc/source/overview/saving.md +++ b/doc/source/overview/saving.md @@ -127,13 +127,17 @@ documented below. ### TensorFlow models -Alibi Detect supports any TensorFlow model that can be serialized to the +Alibi Detect supports serialization of any TensorFlow model that can be serialized to the [HDF5](https://www.tensorflow.org/guide/keras/save_and_serialize#keras_h5_format) format. Custom objects should be pre-registered with [register_keras_serializable](https://www.tensorflow.org/api_docs/python/tf/keras/utils/register_keras_serializable). -``` -%### PyTorch +### Scikit-learn -%### scikit-learn +Scikit-learn models are serialized using [joblib](https://joblib.readthedocs.io/en/latest/persistence.html). +Any scikit-learn model that is a subclass of {py:class}`sklearn.base.BaseEstimator` is supported, including +[xgboost](https://xgboost.readthedocs.io/en/latest/python/python_api.html#module-xgboost.sklearn) models following +the scikit-learn API. + +%### PyTorch diff --git a/requirements/dev.txt b/requirements/dev.txt index f34e27622..1a4d10dee 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -23,3 +23,4 @@ tox>=3.21.0, <4.0.0 # used to generate licence info via `make licenses` twine>3.2.0, <4.0.0 # 4.x causes deps clashes with testing/requirements.txt, as requires rich>=12.0.0 -> requires typing-extensions>=4.0.0 -> too high for spacy and thinc! packaging>=19.0, <22.0 # Used to check scipy version for CVMDrift test. Can be removed once python 3.6 support dropped (and scipy lower bound >=1.7.0). codecov>=2.0.15, <3.0.0 +xgboost>=1.3.2, <2.0.0 # Install for use in testing since we support serialization of xgboost models under the sklearn API From 6b8b1c8953ba6c922a41eeef5c31ab9743460f05 Mon Sep 17 00:00:00 2001 From: SangamSwadik <35230623+SangamSwadiK@users.noreply.github.com> Date: Thu, 13 Oct 2022 15:21:05 +0530 Subject: [PATCH 34/35] fix typo (#651) fix typo in readme examples --- examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index 409059976..c0f053119 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,7 +1,7 @@ Example notebooks are stored in `../doc/source/examples/`. This directory contains symbolic links to the notebooks, which serve as shortcuts. For example, -the `cd_mol.ipynb` notebook can be opened by running `jupyer-notebook examples/cd_mol.ipynb` in +the `cd_mol.ipynb` notebook can be opened by running `jupyter-notebook examples/cd_mol.ipynb` in the root directory. Note: The symbolic links might not work on some Windows versions. From 6df9bd4b0fb6b27a1df41f0cdae76c804dc3cd55 Mon Sep 17 00:00:00 2001 From: Ashley Scillitoe Date: Thu, 13 Oct 2022 15:12:19 +0100 Subject: [PATCH 35/35] Follow-up to sklearn serialization (#650) --- alibi_detect/cd/classifier.py | 12 +-- alibi_detect/cd/context_aware.py | 10 +-- alibi_detect/cd/keops/learned_kernel.py | 3 +- alibi_detect/cd/keops/mmd.py | 5 +- alibi_detect/cd/learned_kernel.py | 12 +-- alibi_detect/cd/lsdd.py | 8 +- alibi_detect/cd/lsdd_online.py | 8 +- alibi_detect/cd/mmd.py | 16 ++-- alibi_detect/cd/mmd_online.py | 10 +-- alibi_detect/cd/model_uncertainty.py | 16 ++-- alibi_detect/cd/pytorch/classifier.py | 3 +- alibi_detect/cd/pytorch/context_aware.py | 3 +- alibi_detect/cd/pytorch/learned_kernel.py | 3 +- alibi_detect/cd/pytorch/lsdd.py | 3 +- alibi_detect/cd/pytorch/lsdd_online.py | 3 +- alibi_detect/cd/pytorch/mmd.py | 3 +- alibi_detect/cd/pytorch/mmd_online.py | 3 +- alibi_detect/cd/sklearn/classifier.py | 3 +- alibi_detect/cd/spot_the_diff.py | 8 +- alibi_detect/cd/tensorflow/classifier.py | 3 +- alibi_detect/cd/tensorflow/context_aware.py | 3 +- alibi_detect/cd/tensorflow/learned_kernel.py | 3 +- alibi_detect/cd/tensorflow/lsdd.py | 3 +- alibi_detect/cd/tensorflow/lsdd_online.py | 3 +- alibi_detect/cd/tensorflow/mmd.py | 3 +- alibi_detect/cd/tensorflow/mmd_online.py | 3 +- alibi_detect/cd/utils.py | 5 +- alibi_detect/saving/_sklearn/saving.py | 5 +- alibi_detect/saving/_tensorflow/loading.py | 3 +- alibi_detect/saving/_tensorflow/saving.py | 5 +- alibi_detect/saving/loading.py | 24 +++--- alibi_detect/saving/saving.py | 13 ++-- alibi_detect/saving/schemas.py | 78 +++++++++++--------- alibi_detect/saving/tests/test_validate.py | 2 +- alibi_detect/utils/_types.py | 18 +++++ alibi_detect/utils/frameworks.py | 9 +++ 36 files changed, 187 insertions(+), 128 deletions(-) diff --git a/alibi_detect/cd/classifier.py b/alibi_detect/cd/classifier.py index 89b5277ae..180320931 100644 --- a/alibi_detect/cd/classifier.py +++ b/alibi_detect/cd/classifier.py @@ -1,7 +1,7 @@ import numpy as np from typing import Callable, Dict, Optional, Union from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, \ - BackendValidator + BackendValidator, Framework from alibi_detect.base import DriftConfigMixin @@ -149,9 +149,9 @@ def __init__( backend = backend.lower() BackendValidator( - backend_options={'tensorflow': ['tensorflow'], - 'pytorch': ['pytorch'], - 'sklearn': ['sklearn']}, + backend_options={Framework.TENSORFLOW: [Framework.TENSORFLOW], + Framework.PYTORCH: [Framework.PYTORCH], + Framework.SKLEARN: [Framework.SKLEARN]}, construct_name=self.__class__.__name__ ).verify_backend(backend) @@ -162,13 +162,13 @@ def __init__( pop_kwargs += ['optimizer'] [kwargs.pop(k, None) for k in pop_kwargs] - if backend == 'tensorflow': + if backend == Framework.TENSORFLOW: pop_kwargs = ['device', 'dataloader', 'use_calibration', 'calibration_kwargs', 'use_oob'] [kwargs.pop(k, None) for k in pop_kwargs] if dataset is None: kwargs.update({'dataset': TFDataset}) self._detector = ClassifierDriftTF(*args, **kwargs) # type: ignore - elif backend == 'pytorch': + elif backend == Framework.PYTORCH: pop_kwargs = ['use_calibration', 'calibration_kwargs', 'use_oob'] [kwargs.pop(k, None) for k in pop_kwargs] if dataset is None: diff --git a/alibi_detect/cd/context_aware.py b/alibi_detect/cd/context_aware.py index 6bc455c4b..bb02c2ad3 100644 --- a/alibi_detect/cd/context_aware.py +++ b/alibi_detect/cd/context_aware.py @@ -1,7 +1,7 @@ import logging import numpy as np from typing import Callable, Dict, Optional, Union, Tuple -from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, BackendValidator +from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, BackendValidator, Framework from alibi_detect.utils.warnings import deprecated_alias from alibi_detect.base import DriftConfigMixin @@ -93,8 +93,8 @@ def __init__( backend = backend.lower() BackendValidator( - backend_options={'tensorflow': ['tensorflow'], - 'pytorch': ['pytorch']}, + backend_options={Framework.TENSORFLOW: [Framework.TENSORFLOW], + Framework.PYTORCH: [Framework.PYTORCH]}, construct_name=self.__class__.__name__ ).verify_backend(backend) @@ -104,7 +104,7 @@ def __init__( [kwargs.pop(k, None) for k in pop_kwargs] if x_kernel is None or c_kernel is None: - if backend == 'tensorflow': + if backend == Framework.TENSORFLOW: from alibi_detect.utils.tensorflow.kernels import GaussianRBF else: from alibi_detect.utils.pytorch.kernels import GaussianRBF # type: ignore[no-redef] @@ -113,7 +113,7 @@ def __init__( if c_kernel is None: kwargs.update({'c_kernel': GaussianRBF}) - if backend == 'tensorflow': + if backend == Framework.TENSORFLOW: kwargs.pop('device', None) self._detector = ContextMMDDriftTF(*args, **kwargs) # type: ignore else: diff --git a/alibi_detect/cd/keops/learned_kernel.py b/alibi_detect/cd/keops/learned_kernel.py index acb9a4f43..e3073713d 100644 --- a/alibi_detect/cd/keops/learned_kernel.py +++ b/alibi_detect/cd/keops/learned_kernel.py @@ -10,6 +10,7 @@ from alibi_detect.cd.base import BaseLearnedKernelDrift from alibi_detect.utils.pytorch import get_device, predict_batch from alibi_detect.utils.pytorch.data import TorchDataset +from alibi_detect.utils.frameworks import Framework class LearnedKernelDriftKeops(BaseLearnedKernelDrift): @@ -130,7 +131,7 @@ def __init__( input_shape=input_shape, data_type=data_type ) - self.meta.update({'backend': 'keops'}) + self.meta.update({'backend': Framework.KEOPS.value}) # Set device, define model and training kwargs self.device = get_device(device) diff --git a/alibi_detect/cd/keops/mmd.py b/alibi_detect/cd/keops/mmd.py index 86173ad13..5b1a2fdc0 100644 --- a/alibi_detect/cd/keops/mmd.py +++ b/alibi_detect/cd/keops/mmd.py @@ -6,6 +6,7 @@ from alibi_detect.cd.base import BaseMMDDrift from alibi_detect.utils.keops.kernels import GaussianRBF from alibi_detect.utils.pytorch import get_device +from alibi_detect.utils.frameworks import Framework logger = logging.getLogger(__name__) @@ -82,7 +83,7 @@ def __init__( input_shape=input_shape, data_type=data_type ) - self.meta.update({'backend': 'keops'}) + self.meta.update({'backend': Framework.KEOPS.value}) # set device self.device = get_device(device) @@ -168,7 +169,7 @@ def score(self, x: Union[np.ndarray, list]) -> Tuple[float, float, float]: x_ref = torch.from_numpy(x_ref).float() # type: ignore[assignment] x = torch.from_numpy(x).float() # type: ignore[assignment] # compute kernel matrix, MMD^2 and apply permutation test - m, n = x_ref.shape[0], x.shape[0] + m, n = x_ref.shape[0], x.shape[0] # type: ignore[union-attr] perms = [torch.randperm(m + n) for _ in range(self.n_permutations)] # TODO - Rethink typings (related to https://github.com/SeldonIO/alibi-detect/issues/540) x_all = torch.cat([x_ref, x], 0) # type: ignore[list-item] diff --git a/alibi_detect/cd/learned_kernel.py b/alibi_detect/cd/learned_kernel.py index 81e3110c4..59288c7ed 100644 --- a/alibi_detect/cd/learned_kernel.py +++ b/alibi_detect/cd/learned_kernel.py @@ -1,6 +1,6 @@ import numpy as np from typing import Callable, Dict, Optional, Union -from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, has_keops, BackendValidator +from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, has_keops, BackendValidator, Framework from alibi_detect.utils.warnings import deprecated_alias from alibi_detect.base import DriftConfigMixin @@ -131,9 +131,9 @@ def __init__( backend = backend.lower() BackendValidator( - backend_options={'tensorflow': ['tensorflow'], - 'pytorch': ['pytorch'], - 'keops': ['keops']}, + backend_options={Framework.TENSORFLOW: [Framework.TENSORFLOW], + Framework.PYTORCH: [Framework.PYTORCH], + Framework.KEOPS: [Framework.KEOPS]}, construct_name=self.__class__.__name__ ).verify_backend(backend) @@ -144,7 +144,7 @@ def __init__( pop_kwargs += ['optimizer'] [kwargs.pop(k, None) for k in pop_kwargs] - if backend == 'tensorflow': + if backend == Framework.TENSORFLOW: pop_kwargs = ['device', 'dataloader', 'batch_size_permutations', 'batch_size_predict'] [kwargs.pop(k, None) for k in pop_kwargs] if dataset is None: @@ -155,7 +155,7 @@ def __init__( kwargs.update({'dataset': TorchDataset}) if dataloader is None: kwargs.update({'dataloader': DataLoader}) - if backend == 'pytorch': + if backend == Framework.PYTORCH: pop_kwargs = ['batch_size_permutations', 'batch_size_predict'] [kwargs.pop(k, None) for k in pop_kwargs] detector = LearnedKernelDriftTorch diff --git a/alibi_detect/cd/lsdd.py b/alibi_detect/cd/lsdd.py index 8335dd3a2..e8a45d30f 100644 --- a/alibi_detect/cd/lsdd.py +++ b/alibi_detect/cd/lsdd.py @@ -1,6 +1,6 @@ import numpy as np from typing import Callable, Dict, Optional, Union, Tuple -from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, BackendValidator +from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, BackendValidator, Framework from alibi_detect.utils.warnings import deprecated_alias from alibi_detect.base import DriftConfigMixin @@ -82,8 +82,8 @@ def __init__( backend = backend.lower() BackendValidator( - backend_options={'tensorflow': ['tensorflow'], - 'pytorch': ['pytorch']}, + backend_options={Framework.TENSORFLOW: [Framework.TENSORFLOW], + Framework.PYTORCH: [Framework.PYTORCH]}, construct_name=self.__class__.__name__ ).verify_backend(backend) @@ -92,7 +92,7 @@ def __init__( pop_kwargs = ['self', 'x_ref', 'backend', '__class__'] [kwargs.pop(k, None) for k in pop_kwargs] - if backend == 'tensorflow': + if backend == Framework.TENSORFLOW: kwargs.pop('device', None) self._detector = LSDDDriftTF(*args, **kwargs) # type: ignore else: diff --git a/alibi_detect/cd/lsdd_online.py b/alibi_detect/cd/lsdd_online.py index 45c551d17..d8d3d5bf6 100644 --- a/alibi_detect/cd/lsdd_online.py +++ b/alibi_detect/cd/lsdd_online.py @@ -1,6 +1,6 @@ import numpy as np from typing import Any, Callable, Dict, Optional, Union -from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, BackendValidator +from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, BackendValidator, Framework from alibi_detect.base import DriftConfigMixin if has_pytorch: from alibi_detect.cd.pytorch.lsdd_online import LSDDDriftOnlineTorch @@ -83,8 +83,8 @@ def __init__( backend = backend.lower() BackendValidator( - backend_options={'tensorflow': ['tensorflow'], - 'pytorch': ['pytorch']}, + backend_options={Framework.TENSORFLOW: [Framework.TENSORFLOW], + Framework.PYTORCH: [Framework.PYTORCH]}, construct_name=self.__class__.__name__ ).verify_backend(backend) @@ -93,7 +93,7 @@ def __init__( pop_kwargs = ['self', 'x_ref', 'ert', 'window_size', 'backend', '__class__'] [kwargs.pop(k, None) for k in pop_kwargs] - if backend == 'tensorflow': + if backend == Framework.TENSORFLOW: kwargs.pop('device', None) self._detector = LSDDDriftOnlineTF(*args, **kwargs) # type: ignore else: diff --git a/alibi_detect/cd/mmd.py b/alibi_detect/cd/mmd.py index b00c20449..3a0c289a5 100644 --- a/alibi_detect/cd/mmd.py +++ b/alibi_detect/cd/mmd.py @@ -1,7 +1,7 @@ import logging import numpy as np from typing import Callable, Dict, Optional, Union, Tuple -from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, has_keops, BackendValidator +from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, has_keops, BackendValidator, Framework from alibi_detect.utils.warnings import deprecated_alias from alibi_detect.base import DriftConfigMixin @@ -88,19 +88,19 @@ def __init__( backend = backend.lower() BackendValidator( - backend_options={'tensorflow': ['tensorflow'], - 'pytorch': ['pytorch'], - 'keops': ['keops']}, + backend_options={Framework.TENSORFLOW: [Framework.TENSORFLOW], + Framework.PYTORCH: [Framework.PYTORCH], + Framework.KEOPS: [Framework.KEOPS]}, construct_name=self.__class__.__name__ ).verify_backend(backend) kwargs = locals() args = [kwargs['x_ref']] pop_kwargs = ['self', 'x_ref', 'backend', '__class__'] - if backend == 'tensorflow': + if backend == Framework.TENSORFLOW: pop_kwargs += ['device', 'batch_size_permutations'] detector = MMDDriftTF - elif backend == 'pytorch': + elif backend == Framework.PYTORCH: pop_kwargs += ['batch_size_permutations'] detector = MMDDriftTorch else: @@ -108,9 +108,9 @@ def __init__( [kwargs.pop(k, None) for k in pop_kwargs] if kernel is None: - if backend == 'tensorflow': + if backend == Framework.TENSORFLOW: from alibi_detect.utils.tensorflow.kernels import GaussianRBF - elif backend == 'pytorch': + elif backend == Framework.PYTORCH: from alibi_detect.utils.pytorch.kernels import GaussianRBF # type: ignore else: from alibi_detect.utils.keops.kernels import GaussianRBF # type: ignore diff --git a/alibi_detect/cd/mmd_online.py b/alibi_detect/cd/mmd_online.py index 409473d43..075877fe8 100644 --- a/alibi_detect/cd/mmd_online.py +++ b/alibi_detect/cd/mmd_online.py @@ -1,6 +1,6 @@ import numpy as np from typing import Any, Callable, Dict, Optional, Union -from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, BackendValidator +from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, BackendValidator, Framework from alibi_detect.base import DriftConfigMixin if has_pytorch: @@ -76,8 +76,8 @@ def __init__( backend = backend.lower() BackendValidator( - backend_options={'tensorflow': ['tensorflow'], - 'pytorch': ['pytorch']}, + backend_options={Framework.TENSORFLOW: [Framework.TENSORFLOW], + Framework.PYTORCH: [Framework.PYTORCH]}, construct_name=self.__class__.__name__ ).verify_backend(backend) @@ -87,13 +87,13 @@ def __init__( [kwargs.pop(k, None) for k in pop_kwargs] if kernel is None: - if backend == 'tensorflow': + if backend == Framework.TENSORFLOW: from alibi_detect.utils.tensorflow.kernels import GaussianRBF else: from alibi_detect.utils.pytorch.kernels import GaussianRBF # type: ignore kwargs.update({'kernel': GaussianRBF}) - if backend == 'tensorflow': + if backend == Framework.TENSORFLOW: kwargs.pop('device', None) self._detector = MMDDriftOnlineTF(*args, **kwargs) # type: ignore else: diff --git a/alibi_detect/cd/model_uncertainty.py b/alibi_detect/cd/model_uncertainty.py index 9804d7c55..fb2527b83 100644 --- a/alibi_detect/cd/model_uncertainty.py +++ b/alibi_detect/cd/model_uncertainty.py @@ -6,7 +6,7 @@ from alibi_detect.cd.chisquare import ChiSquareDrift from alibi_detect.cd.preprocess import classifier_uncertainty, regressor_uncertainty from alibi_detect.cd.utils import encompass_batching, encompass_shuffling_and_batch_filling -from alibi_detect.utils.frameworks import BackendValidator +from alibi_detect.utils.frameworks import BackendValidator, Framework from alibi_detect.base import DriftConfigMixin logger = logging.getLogger(__name__) @@ -85,8 +85,8 @@ def __init__( if backend: backend = backend.lower() - BackendValidator(backend_options={'tensorflow': ['tensorflow'], - 'pytorch': ['pytorch'], + BackendValidator(backend_options={Framework.TENSORFLOW: [Framework.TENSORFLOW], + Framework.PYTORCH: [Framework.PYTORCH], None: []}, construct_name=self.__class__.__name__).verify_backend(backend) @@ -238,8 +238,8 @@ def __init__( if backend: backend = backend.lower() - BackendValidator(backend_options={'tensorflow': ['tensorflow'], - 'pytorch': ['pytorch'], + BackendValidator(backend_options={Framework.TENSORFLOW: [Framework.TENSORFLOW], + Framework.PYTORCH: [Framework.PYTORCH], None: []}, construct_name=self.__class__.__name__).verify_backend(backend) @@ -247,10 +247,10 @@ def __init__( model_fn = model else: if uncertainty_type == 'mc_dropout': - if backend == 'pytorch': + if backend == Framework.PYTORCH: from alibi_detect.cd.pytorch.utils import activate_train_mode_for_dropout_layers model = activate_train_mode_for_dropout_layers(model) - elif backend == 'tensorflow': + elif backend == Framework.TENSORFLOW: logger.warning( "MC dropout being applied to tensorflow model. May not be suitable if model contains" "non-dropout layers with different train and inference time behaviour" @@ -268,7 +268,7 @@ def __init__( max_len=max_len ) - if uncertainty_type == 'mc_dropout' and backend == 'tensorflow': + if uncertainty_type == 'mc_dropout' and backend == Framework.TENSORFLOW: # To average over possible batchnorm effects as all layers evaluated in training mode. model_fn = encompass_shuffling_and_batch_filling(model_fn, batch_size=batch_size) diff --git a/alibi_detect/cd/pytorch/classifier.py b/alibi_detect/cd/pytorch/classifier.py index da0a59d8b..ae77e2471 100644 --- a/alibi_detect/cd/pytorch/classifier.py +++ b/alibi_detect/cd/pytorch/classifier.py @@ -12,6 +12,7 @@ from alibi_detect.utils.pytorch.data import TorchDataset from alibi_detect.utils.pytorch.prediction import predict_batch from alibi_detect.utils.warnings import deprecated_alias +from alibi_detect.utils.frameworks import Framework class ClassifierDriftTorch(BaseClassifierDrift): @@ -138,7 +139,7 @@ def __init__( if preds_type not in ['probs', 'logits']: raise ValueError("'preds_type' should be 'probs' or 'logits'") - self.meta.update({'backend': 'pytorch'}) + self.meta.update({'backend': Framework.PYTORCH.value}) # set device, define model and training kwargs self.device = get_device(device) diff --git a/alibi_detect/cd/pytorch/context_aware.py b/alibi_detect/cd/pytorch/context_aware.py index 337b41922..7b63357ee 100644 --- a/alibi_detect/cd/pytorch/context_aware.py +++ b/alibi_detect/cd/pytorch/context_aware.py @@ -6,6 +6,7 @@ from alibi_detect.utils.pytorch import get_device from alibi_detect.utils.pytorch.kernels import GaussianRBF from alibi_detect.utils.warnings import deprecated_alias +from alibi_detect.utils.frameworks import Framework from alibi_detect.cd._domain_clf import _SVCDomainClf from tqdm import tqdm @@ -101,7 +102,7 @@ def __init__( data_type=data_type, verbose=verbose, ) - self.meta.update({'backend': 'pytorch'}) + self.meta.update({'backend': Framework.PYTORCH.value}) # set device self.device = get_device(device) diff --git a/alibi_detect/cd/pytorch/learned_kernel.py b/alibi_detect/cd/pytorch/learned_kernel.py index bc1ad59d9..2178d018c 100644 --- a/alibi_detect/cd/pytorch/learned_kernel.py +++ b/alibi_detect/cd/pytorch/learned_kernel.py @@ -11,6 +11,7 @@ from alibi_detect.utils.pytorch.distance import mmd2_from_kernel_matrix, batch_compute_kernel_matrix from alibi_detect.utils.pytorch.data import TorchDataset from alibi_detect.utils.warnings import deprecated_alias +from alibi_detect.utils.frameworks import Framework class LearnedKernelDriftTorch(BaseLearnedKernelDrift): @@ -124,7 +125,7 @@ def __init__( input_shape=input_shape, data_type=data_type ) - self.meta.update({'backend': 'pytorch'}) + self.meta.update({'backend': Framework.PYTORCH.value}) # set device, define model and training kwargs self.device = get_device(device) diff --git a/alibi_detect/cd/pytorch/lsdd.py b/alibi_detect/cd/pytorch/lsdd.py index 953f6381a..d62bcf60c 100644 --- a/alibi_detect/cd/pytorch/lsdd.py +++ b/alibi_detect/cd/pytorch/lsdd.py @@ -6,6 +6,7 @@ from alibi_detect.utils.pytorch.kernels import GaussianRBF from alibi_detect.utils.pytorch.distance import permed_lsdds from alibi_detect.utils.warnings import deprecated_alias +from alibi_detect.utils.frameworks import Framework class LSDDDriftTorch(BaseLSDDDrift): @@ -83,7 +84,7 @@ def __init__( input_shape=input_shape, data_type=data_type ) - self.meta.update({'backend': 'pytorch'}) + self.meta.update({'backend': Framework.PYTORCH.value}) # set device self.device = get_device(device) diff --git a/alibi_detect/cd/pytorch/lsdd_online.py b/alibi_detect/cd/pytorch/lsdd_online.py index 256cda282..92c1401c7 100644 --- a/alibi_detect/cd/pytorch/lsdd_online.py +++ b/alibi_detect/cd/pytorch/lsdd_online.py @@ -5,6 +5,7 @@ from alibi_detect.cd.base_online import BaseMultiDriftOnline from alibi_detect.utils.pytorch import get_device from alibi_detect.utils.pytorch import GaussianRBF, permed_lsdds, quantile +from alibi_detect.utils.frameworks import Framework class LSDDDriftOnlineTorch(BaseMultiDriftOnline): @@ -82,7 +83,7 @@ def __init__( input_shape=input_shape, data_type=data_type ) - self.meta.update({'backend': 'pytorch'}) + self.meta.update({'backend': Framework.PYTORCH.value}) self.n_kernel_centers = n_kernel_centers self.lambda_rd_max = lambda_rd_max diff --git a/alibi_detect/cd/pytorch/mmd.py b/alibi_detect/cd/pytorch/mmd.py index 6f279d09b..666942b6c 100644 --- a/alibi_detect/cd/pytorch/mmd.py +++ b/alibi_detect/cd/pytorch/mmd.py @@ -7,6 +7,7 @@ from alibi_detect.utils.pytorch.distance import mmd2_from_kernel_matrix from alibi_detect.utils.pytorch.kernels import GaussianRBF from alibi_detect.utils.warnings import deprecated_alias +from alibi_detect.utils.frameworks import Framework logger = logging.getLogger(__name__) @@ -81,7 +82,7 @@ def __init__( input_shape=input_shape, data_type=data_type ) - self.meta.update({'backend': 'pytorch'}) + self.meta.update({'backend': Framework.PYTORCH.value}) # set device self.device = get_device(device) diff --git a/alibi_detect/cd/pytorch/mmd_online.py b/alibi_detect/cd/pytorch/mmd_online.py index 3ea77c165..808fe5c5d 100644 --- a/alibi_detect/cd/pytorch/mmd_online.py +++ b/alibi_detect/cd/pytorch/mmd_online.py @@ -6,6 +6,7 @@ from alibi_detect.utils.pytorch import get_device from alibi_detect.utils.pytorch.kernels import GaussianRBF from alibi_detect.utils.pytorch import zero_diag, quantile +from alibi_detect.utils.frameworks import Framework class MMDDriftOnlineTorch(BaseMultiDriftOnline): @@ -75,7 +76,7 @@ def __init__( input_shape=input_shape, data_type=data_type ) - self.meta.update({'backend': 'pytorch'}) + self.meta.update({'backend': Framework.PYTORCH.value}) # set device self.device = get_device(device) diff --git a/alibi_detect/cd/sklearn/classifier.py b/alibi_detect/cd/sklearn/classifier.py index 29db0b45f..c2ff1c5cc 100644 --- a/alibi_detect/cd/sklearn/classifier.py +++ b/alibi_detect/cd/sklearn/classifier.py @@ -8,6 +8,7 @@ from sklearn.ensemble import RandomForestClassifier from alibi_detect.cd.base import BaseClassifierDrift from alibi_detect.utils.warnings import deprecated_alias +from alibi_detect.utils.frameworks import Framework logger = logging.getLogger(__name__) @@ -112,7 +113,7 @@ def __init__( if preds_type not in ['probs', 'scores']: raise ValueError("'preds_type' should be 'probs' or 'scores'") - self.meta.update({'backend': 'sklearn'}) + self.meta.update({'backend': Framework.SKLEARN.value}) self.original_model = model self.use_calibration = use_calibration self.calibration_kwargs = dict() if calibration_kwargs is None else calibration_kwargs diff --git a/alibi_detect/cd/spot_the_diff.py b/alibi_detect/cd/spot_the_diff.py index d71e85ead..9c5c63a92 100644 --- a/alibi_detect/cd/spot_the_diff.py +++ b/alibi_detect/cd/spot_the_diff.py @@ -1,6 +1,6 @@ import numpy as np from typing import Callable, Dict, Optional, Union -from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, BackendValidator +from alibi_detect.utils.frameworks import has_pytorch, has_tensorflow, BackendValidator, Framework from alibi_detect.base import DriftConfigMixin if has_pytorch: @@ -127,8 +127,8 @@ def __init__( backend = backend.lower() BackendValidator( - backend_options={'tensorflow': ['tensorflow'], - 'pytorch': ['pytorch']}, + backend_options={Framework.TENSORFLOW: [Framework.TENSORFLOW], + Framework.PYTORCH: [Framework.PYTORCH]}, construct_name=self.__class__.__name__ ).verify_backend(backend) kwargs = locals() @@ -138,7 +138,7 @@ def __init__( pop_kwargs += ['optimizer'] [kwargs.pop(k, None) for k in pop_kwargs] - if backend == 'tensorflow': + if backend == Framework.TENSORFLOW: pop_kwargs = ['device', 'dataloader'] [kwargs.pop(k, None) for k in pop_kwargs] if dataset is None: diff --git a/alibi_detect/cd/tensorflow/classifier.py b/alibi_detect/cd/tensorflow/classifier.py index 1930a65e1..0991e4a1f 100644 --- a/alibi_detect/cd/tensorflow/classifier.py +++ b/alibi_detect/cd/tensorflow/classifier.py @@ -10,6 +10,7 @@ from alibi_detect.utils.tensorflow.misc import clone_model from alibi_detect.utils.tensorflow.prediction import predict_batch from alibi_detect.utils.warnings import deprecated_alias +from alibi_detect.utils.frameworks import Framework class ClassifierDriftTF(BaseClassifierDrift): @@ -129,7 +130,7 @@ def __init__( if preds_type not in ['probs', 'logits']: raise ValueError("'preds_type' should be 'probs' or 'logits'") - self.meta.update({'backend': 'tensorflow'}) + self.meta.update({'backend': Framework.TENSORFLOW.value}) # define and compile classifier model self.original_model = model diff --git a/alibi_detect/cd/tensorflow/context_aware.py b/alibi_detect/cd/tensorflow/context_aware.py index 1ee5b94e0..6f9b773e4 100644 --- a/alibi_detect/cd/tensorflow/context_aware.py +++ b/alibi_detect/cd/tensorflow/context_aware.py @@ -6,6 +6,7 @@ from alibi_detect.cd.base import BaseContextMMDDrift from alibi_detect.utils.tensorflow.kernels import GaussianRBF from alibi_detect.utils.warnings import deprecated_alias +from alibi_detect.utils.frameworks import Framework from alibi_detect.cd._domain_clf import _SVCDomainClf from tqdm import tqdm @@ -97,7 +98,7 @@ def __init__( data_type=data_type, verbose=verbose ) - self.meta.update({'backend': 'tensorflow'}) + self.meta.update({'backend': Framework.TENSORFLOW.value}) # initialize kernel self.x_kernel = x_kernel(init_sigma_fn=_sigma_median_diag) if x_kernel == GaussianRBF else x_kernel diff --git a/alibi_detect/cd/tensorflow/learned_kernel.py b/alibi_detect/cd/tensorflow/learned_kernel.py index 9a06f05f6..a838b21f7 100644 --- a/alibi_detect/cd/tensorflow/learned_kernel.py +++ b/alibi_detect/cd/tensorflow/learned_kernel.py @@ -7,6 +7,7 @@ from alibi_detect.utils.tensorflow.misc import clone_model from alibi_detect.utils.tensorflow.distance import mmd2_from_kernel_matrix, batch_compute_kernel_matrix from alibi_detect.utils.warnings import deprecated_alias +from alibi_detect.utils.frameworks import Framework class LearnedKernelDriftTF(BaseLearnedKernelDrift): @@ -113,7 +114,7 @@ def __init__( input_shape=input_shape, data_type=data_type ) - self.meta.update({'backend': 'tensorflow'}) + self.meta.update({'backend': Framework.TENSORFLOW.value}) # define and compile kernel self.original_kernel = kernel diff --git a/alibi_detect/cd/tensorflow/lsdd.py b/alibi_detect/cd/tensorflow/lsdd.py index 404767799..ef0335ae9 100644 --- a/alibi_detect/cd/tensorflow/lsdd.py +++ b/alibi_detect/cd/tensorflow/lsdd.py @@ -5,6 +5,7 @@ from alibi_detect.utils.tensorflow.kernels import GaussianRBF from alibi_detect.utils.tensorflow.distance import permed_lsdds from alibi_detect.utils.warnings import deprecated_alias +from alibi_detect.utils.frameworks import Framework class LSDDDriftTF(BaseLSDDDrift): @@ -78,7 +79,7 @@ def __init__( input_shape=input_shape, data_type=data_type ) - self.meta.update({'backend': 'tensorflow'}) + self.meta.update({'backend': Framework.TENSORFLOW.value}) if self.preprocess_at_init or self.preprocess_fn is None or self.x_ref_preprocessed: x_ref = tf.convert_to_tensor(self.x_ref) diff --git a/alibi_detect/cd/tensorflow/lsdd_online.py b/alibi_detect/cd/tensorflow/lsdd_online.py index 97baea3b9..540884c5f 100644 --- a/alibi_detect/cd/tensorflow/lsdd_online.py +++ b/alibi_detect/cd/tensorflow/lsdd_online.py @@ -4,6 +4,7 @@ from typing import Any, Callable, Optional, Union from alibi_detect.cd.base_online import BaseMultiDriftOnline from alibi_detect.utils.tensorflow import GaussianRBF, quantile, permed_lsdds +from alibi_detect.utils.frameworks import Framework class LSDDDriftOnlineTF(BaseMultiDriftOnline): @@ -77,7 +78,7 @@ def __init__( input_shape=input_shape, data_type=data_type ) - self.meta.update({'backend': 'tensorflow'}) + self.meta.update({'backend': Framework.TENSORFLOW.value}) self.n_kernel_centers = n_kernel_centers self.lambda_rd_max = lambda_rd_max diff --git a/alibi_detect/cd/tensorflow/mmd.py b/alibi_detect/cd/tensorflow/mmd.py index 1de8d908a..977e1d18c 100644 --- a/alibi_detect/cd/tensorflow/mmd.py +++ b/alibi_detect/cd/tensorflow/mmd.py @@ -6,6 +6,7 @@ from alibi_detect.utils.tensorflow.distance import mmd2_from_kernel_matrix from alibi_detect.utils.tensorflow.kernels import GaussianRBF from alibi_detect.utils.warnings import deprecated_alias +from alibi_detect.utils.frameworks import Framework logger = logging.getLogger(__name__) @@ -76,7 +77,7 @@ def __init__( input_shape=input_shape, data_type=data_type ) - self.meta.update({'backend': 'tensorflow'}) + self.meta.update({'backend': Framework.TENSORFLOW.value}) # initialize kernel if isinstance(sigma, np.ndarray): diff --git a/alibi_detect/cd/tensorflow/mmd_online.py b/alibi_detect/cd/tensorflow/mmd_online.py index 8552802a6..3d4a6b57a 100644 --- a/alibi_detect/cd/tensorflow/mmd_online.py +++ b/alibi_detect/cd/tensorflow/mmd_online.py @@ -5,6 +5,7 @@ from alibi_detect.cd.base_online import BaseMultiDriftOnline from alibi_detect.utils.tensorflow.kernels import GaussianRBF from alibi_detect.utils.tensorflow import zero_diag, quantile, subset_matrix +from alibi_detect.utils.frameworks import Framework class MMDDriftOnlineTF(BaseMultiDriftOnline): @@ -70,7 +71,7 @@ def __init__( input_shape=input_shape, data_type=data_type ) - self.meta.update({'backend': 'tensorflow'}) + self.meta.update({'backend': Framework.TENSORFLOW.value}) # initialize kernel if isinstance(sigma, np.ndarray): diff --git a/alibi_detect/cd/utils.py b/alibi_detect/cd/utils.py index 33959c67c..f8f19e9b0 100644 --- a/alibi_detect/cd/utils.py +++ b/alibi_detect/cd/utils.py @@ -4,6 +4,7 @@ import numpy as np from alibi_detect.utils.sampling import reservoir_sampling +from alibi_detect.utils.frameworks import Framework logger = logging.getLogger(__name__) @@ -63,9 +64,9 @@ def encompass_batching( backend = backend.lower() kwargs = {'batch_size': batch_size, 'tokenizer': tokenizer, 'max_len': max_len, 'preprocess_batch_fn': preprocess_batch_fn} - if backend == 'tensorflow': + if backend == Framework.TENSORFLOW: from alibi_detect.cd.tensorflow.preprocess import preprocess_drift - elif backend == 'pytorch': + elif backend == Framework.PYTORCH: from alibi_detect.cd.pytorch.preprocess import preprocess_drift # type: ignore[no-redef] kwargs['device'] = device else: diff --git a/alibi_detect/saving/_sklearn/saving.py b/alibi_detect/saving/_sklearn/saving.py index bbace2388..b903bc2f7 100644 --- a/alibi_detect/saving/_sklearn/saving.py +++ b/alibi_detect/saving/_sklearn/saving.py @@ -2,10 +2,11 @@ import os from pathlib import Path from typing import Union - import joblib from sklearn.base import BaseEstimator +from alibi_detect.utils.frameworks import Framework + logger = logging.getLogger(__name__) @@ -34,7 +35,7 @@ def save_model_config(model: BaseEstimator, filepath = base_path.joinpath(local_path) save_model(model, filepath=filepath, save_dir='model') cfg_model = { - 'flavour': 'sklearn', + 'flavour': Framework.SKLEARN.value, 'src': local_path.joinpath('model') } return cfg_model diff --git a/alibi_detect/saving/_tensorflow/loading.py b/alibi_detect/saving/_tensorflow/loading.py index ec38b0996..0f28bc7e6 100644 --- a/alibi_detect/saving/_tensorflow/loading.py +++ b/alibi_detect/saving/_tensorflow/loading.py @@ -25,6 +25,7 @@ OutlierVAE, OutlierVAEGMM, SpectralResidual) from alibi_detect.od.llr import build_model from alibi_detect.utils.tensorflow.kernels import DeepKernel +from alibi_detect.utils.frameworks import Framework # Below imports are used for legacy loading, and will be removed (or moved to utils/loading.py) in the future from alibi_detect.version import __version__ from alibi_detect.base import Detector @@ -214,7 +215,7 @@ def load_detector_legacy(filepath: Union[str, os.PathLike], suffix: str, **kwarg warnings.warn('Trying to load detector from an older version.' 'This may lead to breaking code or invalid results.') - if 'backend' in list(meta_dict.keys()) and meta_dict['backend'] == 'pytorch': + if 'backend' in list(meta_dict.keys()) and meta_dict['backend'] == Framework.PYTORCH: raise NotImplementedError('Detectors with PyTorch backend are not yet supported.') detector_name = meta_dict['name'] diff --git a/alibi_detect/saving/_tensorflow/saving.py b/alibi_detect/saving/_tensorflow/saving.py index 9705d62de..3630c6861 100644 --- a/alibi_detect/saving/_tensorflow/saving.py +++ b/alibi_detect/saving/_tensorflow/saving.py @@ -22,6 +22,7 @@ from alibi_detect.utils._types import Literal from alibi_detect.utils.tensorflow.kernels import GaussianRBF from alibi_detect.utils.missing_optional_dependency import MissingDependency +from alibi_detect.utils.frameworks import Framework logger = logging.getLogger(__name__) @@ -82,7 +83,7 @@ def save_model_config(model: Callable, filepath = base_path.joinpath(local_path) save_model(model, filepath=filepath, save_dir='model') cfg_model = { - 'flavour': 'tensorflow', + 'flavour': Framework.TENSORFLOW.value, 'src': local_path.joinpath('model') } return cfg_model, cfg_embed @@ -148,7 +149,7 @@ def save_embedding_config(embed: TransformerEmbedding, cfg_embed.update({'type': embed.emb_type}) cfg_embed.update({'layers': embed.hs_emb.keywords['layers']}) cfg_embed.update({'src': local_path}) - cfg_embed.update({'flavour': 'tensorflow'}) + cfg_embed.update({'flavour': Framework.TENSORFLOW.value}) # Save embedding model logger.info('Saving embedding model to {}.'.format(filepath)) diff --git a/alibi_detect/saving/loading.py b/alibi_detect/saving/loading.py index 385dfcb31..0d316731a 100644 --- a/alibi_detect/saving/loading.py +++ b/alibi_detect/saving/loading.py @@ -16,8 +16,8 @@ from alibi_detect.saving._sklearn import load_model_sk from alibi_detect.saving.validate import validate_config from alibi_detect.base import Detector, ConfigurableDetector -from alibi_detect.utils.frameworks import has_tensorflow, has_pytorch -from alibi_detect.saving.schemas import SupportedModels_tf, SupportedModels_torch +from alibi_detect.utils.frameworks import has_tensorflow, has_pytorch, Framework +from alibi_detect.saving.schemas import supported_models_tf, supported_models_torch if TYPE_CHECKING: import tensorflow as tf @@ -130,7 +130,7 @@ def _load_detector_config(filepath: Union[str, os.PathLike]) -> ConfigurableDete # Backend backend = cfg.get('backend', None) - if backend is not None and backend.lower() not in ('tensorflow', 'sklearn'): + if backend is not None and backend.lower() not in (Framework.TENSORFLOW, Framework.SKLEARN): raise NotImplementedError('Loading detectors with pytorch or keops backend is not yet supported.') # Init detector from config @@ -162,7 +162,7 @@ def _init_detector(cfg: dict) -> ConfigurableDetector: return detector -def _load_kernel_config(cfg: dict, backend: str = 'tensorflow') -> Callable: +def _load_kernel_config(cfg: dict, backend: str = Framework.TENSORFLOW) -> Callable: """ Loads a kernel from a kernel config dict. @@ -179,7 +179,7 @@ def _load_kernel_config(cfg: dict, backend: str = 'tensorflow') -> Callable: ------- The kernel. """ - if backend == 'tensorflow': + if backend == Framework.TENSORFLOW: kernel = load_kernel_config_tf(cfg) else: kernel = None @@ -215,10 +215,10 @@ def _load_preprocess_config(cfg: dict) -> Optional[Callable]: emb = kwargs.pop('embedding') # embedding passed to preprocess_drift as `model` therefore remove # Backend specifics - if has_tensorflow and isinstance(model, SupportedModels_tf): + if has_tensorflow and isinstance(model, supported_models_tf): model = prep_model_and_emb_tf(model, emb) kwargs.pop('device') - elif has_pytorch and isinstance(model, SupportedModels_torch): + elif has_pytorch and isinstance(model, supported_models_torch): raise NotImplementedError('Loading preprocess_fn for PyTorch not yet supported.') # device = cfg['device'] # TODO - device should be set already - check # kwargs.update({'model': kwargs['model'].to(device)}) # TODO - need .to(device) here? @@ -264,9 +264,9 @@ def _load_model_config(cfg: dict) -> Callable: raise FileNotFoundError("The `src` field is not a recognised directory. It should be a directory containing " "a compatible model.") - if flavour == 'tensorflow': + if flavour == Framework.TENSORFLOW: model = load_model_tf(src, load_dir='.', custom_objects=custom_obj, layer=layer) - elif flavour == 'sklearn': + elif flavour == Framework.SKLEARN: model = load_model_sk(src) else: raise NotImplementedError('Loading of PyTorch models not currently supported') @@ -291,7 +291,7 @@ def _load_embedding_config(cfg: dict) -> Callable: # TODO: Could type return mo layers = cfg['layers'] typ = cfg['type'] flavour = cfg['flavour'] - if flavour == 'tensorflow': + if flavour == Framework.TENSORFLOW: emb = load_embedding_tf(src, embedding_type=typ, layers=layers) else: raise NotImplementedError('Loading of non-tensorflow embedding models not currently supported') @@ -335,7 +335,7 @@ def _load_optimizer_config(cfg: dict, ------- The loaded optimizer. """ - if backend == 'tensorflow': + if backend == Framework.TENSORFLOW: optimizer = load_optimizer_tf(cfg) else: raise NotImplementedError('Loading of non-tensorflow optimizers not currently supported') @@ -493,7 +493,7 @@ def resolve_config(cfg: dict, config_dir: Optional[Path]) -> dict: # are not resolved into objects here, since they are yet to undergo a further validation step). Instead, only # their components, such as `src`, are resolved above. elif isinstance(src, dict): - backend = cfg.get('backend', 'tensorflow') + backend = cfg.get('backend', Framework.TENSORFLOW) if key[-1] in ('model', 'proj'): obj = _load_model_config(src) elif key[-1] == 'embedding': diff --git a/alibi_detect/saving/saving.py b/alibi_detect/saving/saving.py index 767ad61e9..05e80831e 100644 --- a/alibi_detect/saving/saving.py +++ b/alibi_detect/saving/saving.py @@ -13,7 +13,8 @@ from alibi_detect.saving._typing import VALID_DETECTORS from alibi_detect.saving.loading import _replace, validate_config from alibi_detect.saving.registry import registry -from alibi_detect.saving.schemas import SupportedModels, SupportedModels_tf, SupportedModels_sklearn +from alibi_detect.utils._types import supported_models_all, supported_models_tf, supported_models_sklearn +from alibi_detect.utils.frameworks import Framework from alibi_detect.base import Detector, ConfigurableDetector from alibi_detect.saving._tensorflow import save_detector_legacy, save_model_config_tf from alibi_detect.saving._sklearn import save_model_config_sk @@ -47,7 +48,7 @@ def save_detector( if legacy: warnings.warn('The `legacy` option will be removed in a future version.', DeprecationWarning) - if 'backend' in list(detector.meta.keys()) and detector.meta['backend'] in ['pytorch', 'keops']: + if 'backend' in list(detector.meta.keys()) and detector.meta['backend'] in [Framework.PYTORCH, Framework.KEOPS]: raise NotImplementedError('Saving detectors with pytorch or keops backend is not yet supported.') # TODO: Replace .__args__ w/ typing.get_args() once Python 3.7 dropped (and remove type ignore below) @@ -125,7 +126,7 @@ def _save_detector_config(detector: ConfigurableDetector, filepath: Union[str, o """ # Get backend, input_shape and detector_name backend = detector.meta.get('backend', None) - if backend not in (None, 'tensorflow', 'sklearn'): + if backend not in (None, Framework.TENSORFLOW, Framework.SKLEARN): raise NotImplementedError("Currently, saving is only supported with backend='tensorflow' and 'sklearn'.") detector_name = detector.__class__.__name__ @@ -264,7 +265,7 @@ def _save_preprocess_config(preprocess_fn: Callable, kwargs = {} for k, v in func_kwargs.items(): # Model/embedding - if isinstance(v, SupportedModels): + if isinstance(v, supported_models_all): cfg_model, cfg_embed = _save_model_config(v, filepath, input_shape, local_path) kwargs.update({k: cfg_model}) if cfg_embed is not None: @@ -411,9 +412,9 @@ def _save_model_config(model: Any, ------- A tuple containing the model and embedding config dicts. """ - if isinstance(model, SupportedModels_tf): + if isinstance(model, supported_models_tf): return save_model_config_tf(model, base_path, input_shape, path) - elif isinstance(model, SupportedModels_sklearn): + elif isinstance(model, supported_models_sklearn): return save_model_config_sk(model, base_path, path), None else: raise NotImplementedError("Support for saving the given model is not yet implemented") diff --git a/alibi_detect/saving/schemas.py b/alibi_detect/saving/schemas.py index 722ab3d3a..fc37b3745 100644 --- a/alibi_detect/saving/schemas.py +++ b/alibi_detect/saving/schemas.py @@ -13,31 +13,15 @@ For detector pydantic models, the fields match the corresponding detector's args/kwargs. Refer to the detector's api docs for a full description of each arg/kwarg. """ - from typing import Callable, Dict, List, Optional, Type, Union, Any import numpy as np from pydantic import BaseModel, validator -from sklearn.base import BaseEstimator # import here since sklearn currently a core dep - -from alibi_detect.cd.tensorflow import UAE as UAE_tf -from alibi_detect.cd.tensorflow import HiddenOutput as HiddenOutput_tf -from alibi_detect.utils._types import Literal, NDArray -from alibi_detect.utils.frameworks import has_tensorflow, has_pytorch -# Define supported models for each optional dependency -SupportedModels_tf, SupportedModels_torch, SupportedModels_sklearn = (), (), () # type: ignore -if has_tensorflow: - import tensorflow as tf - SupportedModels_tf = (tf.keras.Model, UAE_tf, HiddenOutput_tf) # type: ignore -if has_pytorch: - # import torch - SupportedModels_torch = () # type: ignore # TODO - fill -# import sklearn -SupportedModels_sklearn = (BaseEstimator, ) # type: ignore - -# Build SupportedModels - a tuple of all possible models for use in isinstance() etc. -SupportedModels = SupportedModels_tf + SupportedModels_torch + SupportedModels_sklearn +from alibi_detect.utils.frameworks import Framework +from alibi_detect.utils._types import (Literal, NDArray, supported_models_all, supported_models_tf, + supported_models_sklearn, supported_models_torch, supported_optimizers_tf, + supported_optimizers_torch, supported_optimizers_all) # Custom validators (defined here for reuse in multiple pydantic models) @@ -49,7 +33,7 @@ def coerce_int2list(value: int) -> List[int]: return value -class SupportedModelsType: +class SupportedModel: """ Pydantic custom type to check the model is one of the supported types (conditional on what optional deps are installed). @@ -61,18 +45,44 @@ def __get_validators__(cls): @classmethod def validate_model(cls, model: Any, values: dict) -> Any: backend = values['backend'] - if backend == 'tensorflow' and not isinstance(model, SupportedModels_tf): - raise TypeError("`backend='tensorflow'` but the `model` doesn't appear to be a TensorFlow supported model.") - elif backend == 'pytorch' and not isinstance(model, SupportedModels_torch): - raise TypeError("`backend='pytorch'` but the `model` doesn't appear to be a TensorFlow supported model.") - elif backend == 'sklearn' and not isinstance(model, SupportedModels_sklearn): - raise TypeError("`backend='sklearn'` but the `model` doesn't appear to be a scikit-learn supported model.") - elif isinstance(model, SupportedModels): # If model supported and no `backend` incompatibility + err_msg = f"`backend={backend}` but the `model` doesn't appear to be a {backend} supported model, "\ + f"or {backend} is not installed." + if backend == Framework.TENSORFLOW and not isinstance(model, supported_models_tf): + raise TypeError(err_msg) + elif backend == Framework.PYTORCH and not isinstance(model, supported_models_torch): + raise TypeError(err_msg) + elif backend == Framework.SKLEARN and not isinstance(model, supported_models_sklearn): + raise TypeError(f"`backend={backend}` but the `model` doesn't appear to be a {backend} supported model.") + elif isinstance(model, supported_models_all): # If model supported and no `backend` incompatibility return model else: # Catch any other unexpected issues raise TypeError('The model is not recognised as a supported type.') +class SupportedOptimizer: + """ + Pydantic custom type to check the optimizer is one of the supported types (conditional on what optional deps + are installed). + """ + @classmethod + def __get_validators__(cls): + yield cls.validate_optimizer + + @classmethod + def validate_optimizer(cls, optimizer: Any, values: dict) -> Any: + backend = values['backend'] + err_msg = f"`backend={backend}` but the `optimizer` doesn't appear to be a {backend} supported model, "\ + f"or {backend} is not installed." + if backend == Framework.TENSORFLOW and not isinstance(optimizer, supported_optimizers_tf): + raise TypeError(err_msg) + elif backend == Framework.PYTORCH and not isinstance(optimizer, supported_optimizers_torch): + raise TypeError(err_msg) + elif isinstance(optimizer, supported_optimizers_all): # If optimizer supported and no `backend` incompatibility + return optimizer + else: # Catch any other unexpected issues + raise TypeError('The model is not recognised as a supported type.') + + # TODO - We could add validator to check `model` and `embedding` type when chained together. Leave this until refactor # of preprocess_drift. @@ -767,7 +777,7 @@ class ClassifierDriftConfigResolved(DriftDetectorConfigResolved): p_val: float = .05 preprocess_at_init: bool = True update_x_ref: Optional[Dict[str, int]] = None - model: Optional[SupportedModelsType] = None + model: Optional[SupportedModel] = None preds_type: Literal['probs', 'logits'] = 'probs' binarize_preds: bool = False reg_loss_fn: Optional[Callable] = None @@ -775,7 +785,7 @@ class ClassifierDriftConfigResolved(DriftDetectorConfigResolved): n_folds: Optional[int] = None retrain_from_scratch: bool = True seed: int = 0 - optimizer: Optional['tf.keras.optimizers.Optimizer'] = None + optimizer: Optional[SupportedOptimizer] = None learning_rate: float = 1e-3 batch_size: int = 32 preprocess_batch_fn: Optional[Callable] = None @@ -838,7 +848,7 @@ class SpotTheDiffDriftConfigResolved(DriftDetectorConfigResolved): n_folds: Optional[int] = None retrain_from_scratch: bool = True seed: int = 0 - optimizer: Optional['tf.keras.optimizers.Optimizer'] = None + optimizer: Optional[SupportedOptimizer] = None learning_rate: float = 1e-3 batch_size: int = 32 preprocess_batch_fn: Optional[Callable] = None @@ -907,7 +917,7 @@ class LearnedKernelDriftConfigResolved(DriftDetectorConfigResolved): reg_loss_fn: Optional[Callable] = None train_size: Optional[float] = .75 retrain_from_scratch: bool = True - optimizer: Optional['tf.keras.optimizers.Optimizer'] = None + optimizer: Optional[SupportedOptimizer] = None learning_rate: float = 1e-3 batch_size: int = 32 batch_size_predict: int = 1000000 @@ -1169,7 +1179,7 @@ class ClassifierUncertaintyDriftConfigResolved(DetectorConfig): """ backend: Literal['tensorflow', 'pytorch'] = 'tensorflow' x_ref: Union[np.ndarray, list] - model: Optional[SupportedModelsType] = None + model: Optional[SupportedModel] = None p_val: float = .05 x_ref_preprocessed: bool = False update_x_ref: Optional[Dict[str, int]] = None @@ -1222,7 +1232,7 @@ class RegressorUncertaintyDriftConfigResolved(DetectorConfig): """ backend: Literal['tensorflow', 'pytorch'] = 'tensorflow' x_ref: Union[np.ndarray, list] - model: Optional[SupportedModelsType] = None + model: Optional[SupportedModel] = None p_val: float = .05 x_ref_preprocessed: bool = False update_x_ref: Optional[Dict[str, int]] = None diff --git a/alibi_detect/saving/tests/test_validate.py b/alibi_detect/saving/tests/test_validate.py index 10871f4ff..21bc3a250 100644 --- a/alibi_detect/saving/tests/test_validate.py +++ b/alibi_detect/saving/tests/test_validate.py @@ -14,7 +14,7 @@ }, 'name': 'MMDDrift', 'x_ref': np.array([[-0.30074928], [1.50240758], [0.43135768], [2.11295779], [0.79684913]]), - 'p_val': 0.05 + 'p_val': 0.05, } # Define a detector config dict without meta (as simple as it gets!) diff --git a/alibi_detect/utils/_types.py b/alibi_detect/utils/_types.py index 6cf5bf09d..5a3a38cbd 100644 --- a/alibi_detect/utils/_types.py +++ b/alibi_detect/utils/_types.py @@ -6,6 +6,8 @@ import numpy as np from numpy.lib import NumpyVersion from pydantic.fields import ModelField +from sklearn.base import BaseEstimator # import here (instead of later) since sklearn currently a core dep +from alibi_detect.utils.frameworks import has_tensorflow, has_pytorch # Literal for typing if sys.version_info >= (3, 8): @@ -51,3 +53,19 @@ def _validate(cls: Type, val: Any, field: ModelField) -> np.ndarray: return np.asarray(val, dtype=dtype_field.type_) else: return np.asarray(val) + + +# Optional dep dependent tuples of types +supported_models_tf, supported_models_torch, supported_models_sklearn = (), (), () # type: ignore +supported_optimizers_tf, supported_optimizers_torch = (), () # type: ignore +if has_tensorflow: + import tensorflow as tf + supported_models_tf = (tf.keras.Model, ) # type: ignore + supported_optimizers_tf = (tf.keras.optimizers.Optimizer, ) # type: ignore +if has_pytorch: + import torch + supported_models_torch = (torch.nn.Module, torch.nn.Sequential) # type: ignore + supported_optimizers_torch = (torch.optim.Optimizer, ) # type: ignore +supported_models_sklearn = (BaseEstimator, ) # type: ignore +supported_models_all = supported_models_tf + supported_models_torch + supported_models_sklearn +supported_optimizers_all = supported_optimizers_tf + supported_optimizers_torch diff --git a/alibi_detect/utils/frameworks.py b/alibi_detect/utils/frameworks.py index 1c708eb43..233f6cf26 100644 --- a/alibi_detect/utils/frameworks.py +++ b/alibi_detect/utils/frameworks.py @@ -1,5 +1,14 @@ from .missing_optional_dependency import ERROR_TYPES from typing import Optional, List, Dict, Iterable +from enum import Enum + + +class Framework(str, Enum): + PYTORCH = 'pytorch' + TENSORFLOW = 'tensorflow' + KEOPS = 'keops' + SKLEARN = 'sklearn' + try: import tensorflow as tf # noqa